diff --git a/CMakeLists.txt b/CMakeLists.txt index f1d0aba..c3f8e28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,39 +3,100 @@ cmake_minimum_required(VERSION 3.20) project(basisu C CXX) -set(CMAKE_CXX_STANDARD 17) -option(STATIC "static linking" FALSE) -option(SAN "sanitize" FALSE) -option(EXAMPLES "build examples" TRUE) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/bin) +# pybind11: allow old Python finder modules without complaining +if (POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) +endif() + +add_compile_definitions($<$:_DEBUG=1>) + +if (CMAKE_SYSTEM_NAME STREQUAL "WASI") + set(BASISU_BUILD_WASM TRUE) +else() + set(BASISU_BUILD_WASM FALSE) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +option(BASISU_STATIC "static linking" FALSE) +option(BASISU_SAN "sanitize" FALSE) +option(BASISU_EXAMPLES "build examples" TRUE) +option(BASISU_WASM_THREADING "Enable WASI threading support" OFF) +option(BASISU_BUILD_PYTHON "Build native Python module via pybind11" OFF) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin) # For MSVC builds default to SSE enabled, and determine if it's a 64-bit (-A x64) vs. 32-bit (-A Win32) build. if (MSVC) - option(SSE "SSE 4.1 support" TRUE) - if ( CMAKE_GENERATOR_PLATFORM STREQUAL Win32 ) - set(BUILD_X64 0) + option(BASISU_SSE "SSE 4.1 support" TRUE) + if ( CMAKE_GENERATOR_PLATFORM STREQUAL Win32 ) + set(BASISU_BUILD_X64 0) else() - set(BUILD_X64 1) + set(BASISU_BUILD_X64 1) endif() - add_compile_options(/W4) + add_compile_options(/W4) else() - option(SSE "SSE 4.1 support" FALSE) - option(BUILD_X64 "build 64-bit" TRUE) + option(BASISU_SSE "SSE 4.1 support" FALSE) + option(BASISU_BUILD_X64 "build 64-bit" TRUE) endif() -option(ZSTD "ZSTD support for KTX2 transcoding/encoding" TRUE) -option(OPENCL "OpenCL support in encoder" FALSE) +option(BASISU_ZSTD "ZSTD support for KTX2 transcoding/encoding" TRUE) +option(BASISU_OPENCL "OpenCL support in encoder" FALSE) + +# Old option to new (BASISU_ prefixed) automatic remapping +foreach(pair + "STATIC;BASISU_STATIC" + "SAN;BASISU_SAN" + "EXAMPLES;BASISU_EXAMPLES" + "WASM_THREADING;BASISU_WASM_THREADING" + "BUILD_PYTHON;BASISU_BUILD_PYTHON" + "BUILD_X64;BASISU_BUILD_X64" + "SSE;BASISU_SSE" + "ZSTD;BASISU_ZSTD" + "OPENCL;BASISU_OPENCL" +) + list(GET pair 0 OLD) + list(GET pair 1 NEW) + + if(DEFINED ${OLD}) + message(WARNING "[BASISU] Legacy option '${OLD}' is deprecated. Use '${NEW}' instead.") + set(${NEW} "${${OLD}}" CACHE BOOL "" FORCE) + endif() +endforeach() + +if (BASISU_BUILD_WASM) + message(STATUS "Configuring for WASM (WASI-SDK)") + + # WASM is always 32-bit + set(BASISU_BUILD_X64 OFF CACHE BOOL "" FORCE) + + # WASM cannot use SSE + set(BASISU_SSE OFF CACHE BOOL "" FORCE) + + # WASM cannot use OpenCL + set(BASISU_OPENCL OFF CACHE BOOL "" FORCE) + + # WASM cannot use static linking + set(BASISU_STATIC OFF CACHE BOOL "" FORCE) + + # WASM cannot use sanitizers + set(BASISU_SAN OFF CACHE BOOL "" FORCE) +endif() -message("Initial BUILD_X64=${BUILD_X64}") message("Initial CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") -message("Initial SSE=${SSE}") -message("Initial ZSTD=${ZSTD}") -message("Initial OPENCL=${OPENCL}") -message("Initial SAN=${SAN}") -message("Initial EXAMPLES=${EXAMPLES}") +message("Initial BASISU_BUILD_X64=${BASISU_BUILD_X64}") +message("Initial BASISU_BUILD_WASM=${BASISU_BUILD_WASM}") +message("Initial BASISU_WASM_THREADING=${BASISU_WASM_THREADING}") +message("Initial BASISU_BUILD_PYTHON=${BASISU_BUILD_PYTHON}") +message("Initial BASISU_SSE=${BASISU_SSE}") +message("Initial BASISU_ZSTD=${BASISU_ZSTD}") +message("Initial BASISU_OPENCL=${BASISU_OPENCL}") +message("Initial BASISU_SAN=${BASISU_SAN}") +message("Initial BASISU_EXAMPLES=${BASISU_EXAMPLES}") -if ((NOT MSVC) AND OPENCL) +if ((NOT MSVC) AND BASISU_OPENCL) # With MSVC builds we use the Khronos lib/include files in the project's "OpenCL" directory, to completely avoid requiring fiddly to install vendor SDK's. # Otherwise we use the system's (if any). find_package(OpenCL) @@ -50,47 +111,47 @@ endif() message(${PROJECT_NAME} " build type: " ${CMAKE_BUILD_TYPE}) -if (BUILD_X64) +if (BASISU_BUILD_X64) message("Building 64-bit") else() message("Building 32-bit") endif() -if (SSE) +if (BASISU_SSE) message("SSE enabled") else() message("SSE disabled") endif() -if (ZSTD) +if (BASISU_ZSTD) message("Zstandard enabled") else() message("Zstandard disabled") endif() -if (NOT MSVC) +if (NOT MSVC AND NOT BASISU_BUILD_WASM) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g") - + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") - if (SAN) - message("Enabling SAN") - - set(SANITIZE_FLAGS "-fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment") - - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${SANITIZE_FLAGS}") + if (BASISU_SAN) + message("Enabling SAN") + + set(SANITIZE_FLAGS "-fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment") + + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${SANITIZE_FLAGS}") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${SANITIZE_FLAGS}") - + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${SANITIZE_FLAGS}") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${SANITIZE_FLAGS}") endif() - set(CMAKE_CXX_FLAGS -std=c++17) - set(GCC_COMPILE_FLAGS "-fvisibility=hidden -fPIC -fno-strict-aliasing -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 -Wall -Wextra -Wno-unused-local-typedefs -Wno-unused-value -Wno-unused-parameter -Wno-unused-variable -Wno-reorder -Wno-misleading-indentation -Wno-class-memaccess -Wno-deprecated-copy -Wno-maybe-uninitialized -Wno-unused-function -Wno-stringop-overflow -Wno-unknown-warning-option") - - if (NOT BUILD_X64) + # set(CMAKE_CXX_FLAGS -std=c++17) + set(GCC_COMPILE_FLAGS "-fvisibility=hidden -fPIC -fno-strict-aliasing -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 -Wall -Wextra -Wno-misleading-indentation") + + if (NOT BASISU_BUILD_X64) set(GCC_COMPILE_FLAGS "${GCC_COMPILE_FLAGS} -m32") endif() @@ -99,25 +160,25 @@ if (NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s ALLOW_MEMORY_GROWTH=1 -DBASISU_SUPPORT_SSE=0") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_LINK_FLAGS}") - elseif (STATIC) - if (SSE) + elseif (BASISU_STATIC) + if (BASISU_SSE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_SSE=1 -msse4.1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_SSE=1 -msse4.1") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_SSE=0") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_SSE=0") endif() - + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_LINK_FLAGS} -static-libgcc -static-libstdc++ -static") else() - if (SSE) + if (BASISU_SSE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_SSE=1 -msse4.1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_SSE=1 -msse4.1") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_SSE=0") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_SSE=0") endif() - + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_LINK_FLAGS} -Wl,-rpath .") endif() @@ -128,8 +189,24 @@ if (NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COMPILE_FLAGS}") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${GCC_COMPILE_FLAGS}") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${GCC_COMPILE_FLAGS} -D_DEBUG") +elseif (BASISU_BUILD_WASM) + # _WASI_EMULATED_PROCESS_CLOCKS/-lwasi-emulated-process-clocks is only for ZStd + add_compile_definitions(_WASI_EMULATED_PROCESS_CLOCKS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lwasi-emulated-process-clocks") + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing -fvisibility=hidden -Wall -Wextra -Wno-unknown-warning-option") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing -fvisibility=hidden -Wall -Wextra -Wno-unknown-warning-option") + + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g") + + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O2") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2") + + # We need a few MB of stack - don't skip this or WASMTime will silently allow the stack to grow into the heap or static memory, causing corruption. + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack-first -Wl,-z,stack-size=8388608") else() - if (SSE) + if (BASISU_SSE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_SSE=1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_SSE=1") else() @@ -160,11 +237,12 @@ set(ENCODER_LIB_SRC_LIST encoder/basisu_uastc_hdr_4x4_enc.cpp encoder/basisu_astc_hdr_6x6_enc.cpp encoder/basisu_astc_hdr_common.cpp + encoder/basisu_astc_ldr_common.cpp + encoder/basisu_astc_ldr_encode.cpp encoder/3rdparty/android_astc_decomp.cpp encoder/3rdparty/tinyexr.cpp transcoder/basisu_transcoder.cpp - - encoder/basisu_astc_hdr_6x6_enc.h + encoder/basisu_astc_hdr_6x6_enc.h encoder/basisu_astc_hdr_common.h encoder/basisu_backend.h encoder/basisu_basis_file.h @@ -186,6 +264,8 @@ set(ENCODER_LIB_SRC_LIST encoder/basisu_ssim.h encoder/basisu_uastc_enc.h encoder/basisu_uastc_hdr_4x4_enc.h + encoder/basisu_astc_ldr_common.h + encoder/basisu_astc_ldr_encode.h encoder/cppspmd_flow.h encoder/cppspmd_math_declares.h encoder/cppspmd_math.h @@ -204,11 +284,12 @@ set(ENCODER_LIB_SRC_LIST transcoder/basisu_transcoder_internal.h transcoder/basisu_transcoder_uastc.h transcoder/basisu_transcoder.h + transcoder/basisu_idct.h transcoder/basisu.h - zstd/zstd.h + zstd/zstd.h ) -if (ZSTD) +if (BASISU_ZSTD) set(ENCODER_LIB_SRC_LIST ${ENCODER_LIB_SRC_LIST} zstd/zstd.c) endif() @@ -220,106 +301,369 @@ add_executable(basisu basisu_tool.cpp) target_link_libraries(basisu PRIVATE basisu_encoder) # Create the new example executable and link against the static library -if(EXAMPLES) - add_executable(examples example/example.cpp) - target_link_libraries(examples PRIVATE basisu_encoder) +if(BASISU_EXAMPLES) + add_executable(example example/example.cpp) + target_link_libraries(example PRIVATE basisu_encoder) + + add_executable(example_capi example_capi/example_capi.c encoder/basisu_wasm_api.cpp encoder/basisu_wasm_transcoder_api.cpp) + target_link_libraries(example_capi PRIVATE basisu_encoder) + + add_executable(example_transcoding example_transcoding/example_transcoding.cpp example_transcoding/utils.cpp zstd/zstddeclib.c transcoder/basisu_transcoder.cpp) +endif() + +if (BASISU_BUILD_WASM) + # Add proper suffix + set_target_properties(basisu PROPERTIES SUFFIX ".wasm") + + if(BASISU_EXAMPLES) + set_target_properties(example PROPERTIES SUFFIX ".wasm") + set_target_properties(example_capi PROPERTIES SUFFIX ".wasm") + set_target_properties(example_transcoding PROPERTIES SUFFIX ".wasm") + endif() + + if (BASISU_WASM_THREADING) + # multithreaded: use "basisu_mt.wasm" + set_target_properties(basisu PROPERTIES OUTPUT_NAME "basisu_mt") + + if(BASISU_EXAMPLES) + set_target_properties(example PROPERTIES OUTPUT_NAME "example_mt") + set_target_properties(example_capi PROPERTIES OUTPUT_NAME "example_capi_mt") + set_target_properties(example_transcoding PROPERTIES OUTPUT_NAME "example_transcoding_mt") + endif() + + else() + # single-threaded: use "basisu_st.wasm" + set_target_properties(basisu PROPERTIES OUTPUT_NAME "basisu_st") + + if(BASISU_EXAMPLES) + set_target_properties(example PROPERTIES OUTPUT_NAME "example_st") + set_target_properties(example_capi PROPERTIES OUTPUT_NAME "example_capi_st") + set_target_properties(example_transcoding PROPERTIES OUTPUT_NAME "example_transcoding_st") + endif() + + endif() + + # 256 MB initial, 3.5 GB max — safe defaults for BasisU + target_link_options(basisu PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + ) + + if (BASISU_EXAMPLES) + target_link_options(example PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + ) + target_link_options(example_capi PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + ) + target_link_options(example_transcoding PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + ) + endif() endif() #if (MSVC) target_sources(basisu PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") - if(EXAMPLES) - target_sources(examples PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") - endif() + if(BASISU_EXAMPLES) + target_sources(example PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") + target_sources(example_capi PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") + target_sources(example_transcoding PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/basisu.manifest") + endif() #endif() -if (ZSTD) +if (BASISU_ZSTD) target_compile_definitions(basisu PRIVATE BASISD_SUPPORT_KTX2_ZSTD=1) - if(EXAMPLES) - target_compile_definitions(examples PRIVATE BASISD_SUPPORT_KTX2_ZSTD=1) - endif() + if(BASISU_EXAMPLES) + target_compile_definitions(example PRIVATE BASISD_SUPPORT_KTX2_ZSTD=1) + target_compile_definitions(example_capi PRIVATE BASISD_SUPPORT_KTX2_ZSTD=1) + target_compile_definitions(example_transcoding PRIVATE BASISD_SUPPORT_KTX2_ZSTD=1) + endif() else() target_compile_definitions(basisu PRIVATE BASISD_SUPPORT_KTX2_ZSTD=0) - if(EXAMPLES) - target_compile_definitions(examples PRIVATE BASISD_SUPPORT_KTX2_ZSTD=0) - endif() + if(BASISU_EXAMPLES) + target_compile_definitions(example PRIVATE BASISD_SUPPORT_KTX2_ZSTD=0) + target_compile_definitions(example_capi PRIVATE BASISD_SUPPORT_KTX2_ZSTD=0) + target_compile_definitions(example_transcoding PRIVATE BASISD_SUPPORT_KTX2_ZSTD=0) + endif() endif() if (NOT MSVC) # For Non-Windows builds, let cmake try and find the system OpenCL headers/libs for us. - if (OPENCL AND OPENCL_FOUND) + if (BASISU_OPENCL AND OPENCL_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_OPENCL=1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_OPENCL=1") - + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_OPENCL=1") + target_include_directories(basisu PRIVATE ${OpenCL_INCLUDE_DIRS}) - if(EXAMPLES) - target_include_directories(examples PRIVATE ${OpenCL_INCLUDE_DIRS}) - endif() - target_include_directories(basisu_encoder PRIVATE ${OpenCL_INCLUDE_DIRS}) + if(BASISU_EXAMPLES) + target_include_directories(example PRIVATE ${OpenCL_INCLUDE_DIRS}) + target_include_directories(example_capi PRIVATE ${OpenCL_INCLUDE_DIRS}) + endif() + target_include_directories(basisu_encoder PRIVATE ${OpenCL_INCLUDE_DIRS}) set(BASISU_EXTRA_LIBS ${OpenCL_LIBRARIES}) endif() else() # For Windows builds, we use our local copies of the OpenCL import lib and Khronos headers. - if (OPENCL) + if (BASISU_OPENCL) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBASISU_SUPPORT_OPENCL=1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_OPENCL=1") - + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBASISU_SUPPORT_OPENCL=1") + target_include_directories(basisu PRIVATE "OpenCL") - if(EXAMPLES) - target_include_directories(examples PRIVATE "OpenCL") - endif() - target_include_directories(basisu_encoder PRIVATE "OpenCL") + if(BASISU_EXAMPLES) + target_include_directories(example PRIVATE "OpenCL") + target_include_directories(example_capi PRIVATE "OpenCL") + endif() + target_include_directories(basisu_encoder PRIVATE "OpenCL") - if (BUILD_X64) + if (BASISU_BUILD_X64) target_link_libraries(basisu PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL64.lib") - if(EXAMPLES) - target_link_libraries(examples PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL64.lib") - endif() + if(BASISU_EXAMPLES) + target_link_libraries(example PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL64.lib") + target_link_libraries(example_capi PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL64.lib") + endif() else() target_link_libraries(basisu PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL.lib") - if(EXAMPLES) - target_link_libraries(examples PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL.lib") - endif() + if(BASISU_EXAMPLES) + target_link_libraries(example PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL.lib") + target_link_libraries(example_capi PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/OpenCL/lib/OpenCL.lib") + endif() endif() endif() +endif() + +if (NOT MSVC AND NOT BASISU_BUILD_WASM) + target_link_libraries(basisu PRIVATE m pthread ${BASISU_EXTRA_LIBS}) + if(BASISU_EXAMPLES) + target_link_libraries(example PRIVATE m pthread ${BASISU_EXTRA_LIBS}) + target_link_libraries(example_capi PRIVATE m pthread ${BASISU_EXTRA_LIBS}) + target_link_libraries(example_transcoding PRIVATE m pthread ${BASISU_EXTRA_LIBS}) + endif() endif() -if (NOT MSVC) - target_link_libraries(basisu PRIVATE m pthread ${BASISU_EXTRA_LIBS}) - if(EXAMPLES) - target_link_libraries(examples PRIVATE m pthread ${BASISU_EXTRA_LIBS}) - endif() -endif() - if (NOT EMSCRIPTEN) - if (UNIX) + if (UNIX AND NOT BASISU_BUILD_WASM) if (CMAKE_BUILD_TYPE STREQUAL Release) if (APPLE) add_custom_command(TARGET basisu POST_BUILD COMMAND strip -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu) - #message("strip command: strip -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu") + #message("strip command: strip -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu") else() add_custom_command(TARGET basisu POST_BUILD COMMAND strip -g -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu) - #message("strip command: strip -g -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu") + #message("strip command: strip -g -X -x ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/basisu") endif() endif() endif() endif() if (MSVC) - set_target_properties(basisu PROPERTIES + set_target_properties(basisu PROPERTIES RUNTIME_OUTPUT_NAME "basisu" RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + + if(BASISU_EXAMPLES) + set_target_properties(example PROPERTIES + RUNTIME_OUTPUT_NAME "example" + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + endif() + + if(BASISU_EXAMPLES) + set_target_properties(example_capi PROPERTIES + RUNTIME_OUTPUT_NAME "example_capi" + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + endif() + + if(BASISU_EXAMPLES) + set_target_properties(example_transcoding PROPERTIES + RUNTIME_OUTPUT_NAME "example_transcoding" + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + endif() +endif() + +# ------------------------------------------------------------ +# Build WASM WASI executables (single or multi-threaded) +# ------------------------------------------------------------ +if (BASISU_BUILD_WASM) + target_compile_options(basisu_encoder PRIVATE -fno-exceptions -fno-rtti) + target_compile_options(basisu PRIVATE -fno-exceptions -fno-rtti) + + if (BASISU_EXAMPLES) + target_compile_options(example PRIVATE -fno-exceptions -fno-rtti) + target_compile_options(example_capi PRIVATE -fno-exceptions -fno-rtti) + target_compile_options(example_transcoding PRIVATE -fno-exceptions -fno-rtti) + endif() +endif() + +if (BASISU_BUILD_WASM AND BASISU_WASM_THREADING) + target_compile_definitions(basisu_encoder PRIVATE BASISU_WASI_THREADS=1) + target_compile_definitions(basisu PRIVATE BASISU_WASI_THREADS=1) + + if (BASISU_EXAMPLES) + target_compile_definitions(example PRIVATE BASISU_WASI_THREADS=1) + target_compile_definitions(example_capi PRIVATE BASISU_WASI_THREADS=1) + target_compile_definitions(example_transcoding PRIVATE BASISU_WASI_THREADS=1) + endif() +endif() + +# ------------------------------------------------------------ +# Build WASM WASI API module (single or multi-threaded) +# ------------------------------------------------------------ +if (BASISU_BUILD_WASM) + set(BASISU_WASM_API_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/encoder/basisu_wasm_api.cpp ) - if(EXAMPLES) - set_target_properties(examples PROPERTIES - RUNTIME_OUTPUT_NAME "examples" - RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} - ) - endif() + # Select output name based on threading flag + if (BASISU_WASM_THREADING) + set(BASISU_WASM_OUTPUT_NAME "basisu_module_mt") + else() + set(BASISU_WASM_OUTPUT_NAME "basisu_module_st") + endif() + + add_executable(${BASISU_WASM_OUTPUT_NAME} ${BASISU_WASM_API_SRC}) + target_link_libraries(${BASISU_WASM_OUTPUT_NAME} PRIVATE basisu_encoder) + + set_target_properties(${BASISU_WASM_OUTPUT_NAME} PROPERTIES SUFFIX ".wasm") + + # Common WASM options + target_link_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -Wl,--initial-memory=268435456 + -Wl,--max-memory=3758096384 + -Wl,--stack-first + -Wl,-z,stack-size=8388608 + ) + target_compile_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE -fno-exceptions -fno-rtti) + + set_target_properties(${BASISU_WASM_OUTPUT_NAME} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py/wasm + ) + + # Threading options + if (BASISU_WASM_THREADING) + target_compile_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -pthread + -matomics + ) + target_link_options(${BASISU_WASM_OUTPUT_NAME} PRIVATE + -pthread + -Wl,--shared-memory + -Wl,--export-memory + ) + + target_compile_definitions(${BASISU_WASM_OUTPUT_NAME} PRIVATE BASISU_WASI_THREADS=1) + endif() + +endif() + +if (BASISU_BUILD_WASM) + # Select output name based on threading flag + if (BASISU_WASM_THREADING) + set(BASISU_TRANSCODER_WASM_OUTPUT_NAME "basisu_transcoder_module_mt") + else() + set(BASISU_TRANSCODER_WASM_OUTPUT_NAME "basisu_transcoder_module_st") + endif() + + add_executable(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} + ${CMAKE_CURRENT_SOURCE_DIR}/encoder/basisu_wasm_transcoder_api.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/transcoder/basisu_transcoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/zstd/zstddeclib.c) + + set_target_properties(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PROPERTIES SUFFIX ".wasm") + + target_link_options(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PRIVATE + -Wl,--initial-memory=16777216 + -Wl,--stack-first + -Wl,-z,stack-size=4194304 + ) + target_compile_options(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PRIVATE -fno-exceptions -fno-rtti) + + set_target_properties(${BASISU_TRANSCODER_WASM_OUTPUT_NAME} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py/wasm + ) +endif() + + +# ------------------------------------------------------------ +# Optional: Build native Python modules (pybind11) +# ------------------------------------------------------------ +if (BASISU_BUILD_PYTHON AND NOT BASISU_BUILD_WASM) + find_package(pybind11 CONFIG REQUIRED) + + message(STATUS "Building pybind11 Python extension: basisu_python") + + pybind11_add_module(basisu_python + python/basisu_encoder_pybind11.cpp + encoder/basisu_wasm_api.cpp + ) + + # Ensure PIC ONLY for this target + set_property(TARGET basisu_python PROPERTY POSITION_INDEPENDENT_CODE ON) + + target_link_libraries(basisu_python PRIVATE basisu_encoder) + + # Put basisu_python so into python/basisu_py + set_target_properties(basisu_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + PREFIX "" # Required by Python + OUTPUT_NAME "basisu_python" # Just to be explicit + ) + + if (MSVC) + set_target_properties(basisu_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + SUFFIX ".pyd" + ) + endif() + +endif() + +if (BASISU_BUILD_PYTHON AND NOT BASISU_BUILD_WASM) + find_package(pybind11 CONFIG REQUIRED) + + message(STATUS "Building pybind11 Python extension: basisu_transcoder_python") + + pybind11_add_module(basisu_transcoder_python + python/basisu_transcoder_pybind11.cpp + encoder/basisu_wasm_transcoder_api.cpp + transcoder/basisu_transcoder.cpp + zstd/zstddeclib.c + ) + + # Ensure PIC ONLY for this target + set_property(TARGET basisu_transcoder_python PROPERTY POSITION_INDEPENDENT_CODE ON) + + set_target_properties(basisu_transcoder_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + PREFIX "" + OUTPUT_NAME "basisu_transcoder_python" + ) + + if (MSVC) + set_target_properties(basisu_transcoder_python PROPERTIES + LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_CURRENT_SOURCE_DIR}/python/basisu_py + SUFFIX ".pyd" + ) + endif() endif() diff --git a/basisu.sln b/basisu.sln index 2309ae5..566f85c 100644 --- a/basisu.sln +++ b/basisu.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.8.34322.80 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11222.15 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "basisu", "basisu.vcxproj", "{59586A07-8E7E-411D-BC3D-387E039AA423}" EndProject @@ -9,6 +9,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example", "example\example. EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "encoder_lib", "encoder_lib\encoder_lib.vcxproj", "{97C34996-F458-4030-A402-B32C581872F1}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example_transcoding", "example_transcoding\example_transcoding.vcxproj", "{13333092-FCFE-4D74-8E76-F10C6037593C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "example_capi", "example_capi\example_capi.vcxproj", "{BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64EC = Debug|ARM64EC @@ -55,6 +59,30 @@ Global {97C34996-F458-4030-A402-B32C581872F1}.Release|x64.Build.0 = Release|x64 {97C34996-F458-4030-A402-B32C581872F1}.Release|x86.ActiveCfg = Release|Win32 {97C34996-F458-4030-A402-B32C581872F1}.Release|x86.Build.0 = Release|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x64.ActiveCfg = Debug|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x64.Build.0 = Debug|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x86.ActiveCfg = Debug|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Debug|x86.Build.0 = Debug|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|ARM64EC.Build.0 = Release|ARM64EC + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x64.ActiveCfg = Release|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x64.Build.0 = Release|x64 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x86.ActiveCfg = Release|Win32 + {13333092-FCFE-4D74-8E76-F10C6037593C}.Release|x86.Build.0 = Release|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x64.ActiveCfg = Debug|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x64.Build.0 = Debug|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x86.ActiveCfg = Debug|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Debug|x86.Build.0 = Debug|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|ARM64EC.Build.0 = Release|ARM64EC + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x64.ActiveCfg = Release|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x64.Build.0 = Release|x64 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x86.ActiveCfg = Release|Win32 + {BE889347-E4FD-47DD-BBF4-81F98FAA8BA9}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/basisu.vcxproj b/basisu.vcxproj index fce24bd..1e52b46 100644 --- a/basisu.vcxproj +++ b/basisu.vcxproj @@ -1,4 +1,4 @@ - + @@ -36,40 +36,40 @@ Application true MultiByte - v143 + v145 Application false true MultiByte - v143 + v145 Application true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application false true Unicode - v143 + v145 @@ -129,7 +129,7 @@ true OpenCL _MBCS;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1;_HAS_EXCEPTIONS=0 - StreamingSIMDExtensions2 + AdvancedVectorExtensions stdcpp17 @@ -147,9 +147,9 @@ true true _MBCS;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1; - StreamingSIMDExtensions2 - Level4 + AdvancedVectorExtensions stdcpp17 + Level4 Console @@ -186,7 +186,7 @@ NDEBUG;_HAS_EXCEPTIONS=0;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1 false AnySuitable - StreamingSIMDExtensions2 + AdvancedVectorExtensions Precise false true @@ -217,7 +217,7 @@ Precise true Speed - StreamingSIMDExtensions2 + AdvancedVectorExtensions false stdcpp17 @@ -266,7 +266,10 @@ {97c34996-f458-4030-a402-b32c581872f1} + + + - + \ No newline at end of file diff --git a/basisu.vcxproj.filters b/basisu.vcxproj.filters index a772c98..5281d48 100644 --- a/basisu.vcxproj.filters +++ b/basisu.vcxproj.filters @@ -1,4 +1,4 @@ - + @@ -6,4 +6,7 @@ - + + + + \ No newline at end of file diff --git a/basisu_tool.cpp b/basisu_tool.cpp index 430f9d6..d607f5f 100644 --- a/basisu_tool.cpp +++ b/basisu_tool.cpp @@ -1,4 +1,4 @@ -// basisu_tool.cpp +// basisu_tool.cpp // Copyright (C) 2019-2025 Binomial LLC. All Rights Reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #if _MSC_VER -// For sprintf(), strcpy() +// For sprintf(), strcpy() #define _CRT_SECURE_NO_WARNINGS (1) #pragma warning(disable:4505) // unreferenced function with internal linkage has been removed #pragma warning(disable:4189) // local variable is initialized but not referenced @@ -30,6 +30,7 @@ #include "transcoder/basisu_transcoder.h" #include "encoder/basisu_ssim.h" #include "encoder/basisu_opencl.h" +#include "encoder/basisu_astc_ldr_common.h" #define MINIZ_HEADER_FILE_ONLY #define MINIZ_NO_ZLIB_COMPATIBLE_NAMES @@ -44,6 +45,7 @@ #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include +// Work around Win11 debug console bug. DO NOT SHIP SET TO 1. #define CLEAR_WIN32_CONSOLE 0 #endif @@ -55,7 +57,21 @@ using namespace basisu; using namespace buminiz; -#define BASISU_TOOL_VERSION "1.60.0" +#define BASISU_TOOL_VERSION "2.00.0" + +#if defined(DEBUG) +#pragma message("DEBUG defined") +#endif + +#if defined(_DEBUG) +#pragma message("_DEBUG defined") +#endif + +#ifndef NDEBUG +#if !defined(DEBUG) && !defined(_DEBUG) +#pragma message("NDEBUG is NOT defined, but DEBUG or _DEBUG are also NOT defined, which isn't ideal as extra debug assertion checks will not be compiled.") +#endif +#endif // Define to lower the -test and -test_hdr tolerances //#define USE_TIGHTER_TEST_TOLERANCES @@ -79,6 +95,7 @@ enum tool_mode cTestHDR_4x4, cTestHDR_6x6, cTestHDR_6x6i, + cTestXUASTCLDR, cCLBench, cSplitImage, cCombineImages, @@ -89,197 +106,8 @@ static void print_usage() { printf("\nUsage: basisu filename [filename ...] \n"); - puts("\n" - "The default processing mode is compression of one or more .PNG/.BMP/.TGA/.JPG/.QOI/.DDS/.EXR/.HDR files to a LDR or HDR .KTX2 file. Alternate modes:\n" - " -unpack: Use transcoder to unpack a .basis/.KTX2 file to one or more .KTX/.PNG files\n" - " -validate: Validate and display information about a .basis/.KTX2 file\n" - " -info: Display high-level information about a .basis/.KTX2 file\n" - " -compare: Compare two LDR PNG/BMP/TGA/JPG/QOI images specified with -file, output PSNR and SSIM statistics and RGB/A delta images\n" - " -compare_hdr: Compare two HDR .EXR/.HDR images specified with -file, output PSNR statistics and RGB delta images\n" - " -tonemap: Tonemap an HDR or EXR image to PNG at multiple exposures, use -file to specify filename\n" - " -version: Print version and exit\n" - "\n" - "--- Notes:\n" - "\nUnless an explicit mode is specified, if one or more files have the .basis or .KTX2 extension this tool defaults to unpack mode.\n" - "\nBy default, the compressor assumes the input is in the sRGB colorspace (like photos/albedo textures).\n" - "If the input is NOT sRGB (like a normal map), be sure to specify -linear for less artifacts. Depending on the content type, some experimentation may be needed.\n" - "\n" - "The TinyEXR library is used to read .EXR images. This library does not support all .EXR compression methods. For unsupported images, you can use ImageMagick to convert them to uncompressed .EXR.\n" - "\n" - "For .DDS source files: Mipmapped or not mipmapped 2D textures (but not cubemaps) are supported. Only uncompressed 32-bit RGBA/BGRA, half float RGBA, or float RGBA .DDS files are supported. In -tex_array mode, if a .DDS file is specified, all source files must be in .DDS format.\n" - "\n" - "Filenames prefixed with a @ symbol are read as filename listing files. Listing text files specify which actual filenames to process (one filename per line).\n" - "\n" - "--- Texture Mode Options:\n" - " -etc1s: Encode to ETC1S LDR (the default for SDR/LDR inputs). Roughly .8-2.5 bpp.\n" - " -uastc: Encode to UASTC LDR 4x4. Roughly 5-8 bpp.\n" - " -hdr/-hdr_4x4: Encode input as UASTC HDR 4x4 (the default if any input file has the .EXR or .HDR extension, or if any .DDS file is HDR). Roughly 5-8 bpp.\n" - " -hdr_6x6: Encode input as RDO or highest quality UASTC HDR 6x6. Use -lambda X (try 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls the quality vs. size tradeoff. Roughly 1.2-3.2 bpp.\n" - " -hdr_6x6i: Encode input as UASTC HDR 6x6 intermediate. Use -lambda X (try 100-20000 or higher) option to enable RDO UASTC HDR 6x6, where x controls the quality vs. size tradeoff. Roughly 1-3.2 bpp.\n" - "\n" - "--- Options:\n" - " -ktx2: Write .KTX2 files (the default). By default, UASTC LDR/HDR 4x4 and ASTC 6x6 files will be compressed using Zstandard unless -ktx2_no_zstandard is specified.\n" - " -basis: Write .basis files instead of .KTX2 files (the previous default).\n" - " -file filename.png/tga/jpg/qoi/exr/hdr: Input image filename, multiple images are OK, use -file X for each input filename (prefixing input filenames with -file is optional)\n" - " -alpha_file filename.png/tga/jpg/qoi: Input alpha image filename, multiple images are OK, use -file X for each input filename (must be paired with -file), images converted to REC709 grayscale and used as input alpha\n" - " -output_file filename: Output .basis/.KTX2 filename\n" - " -output_path: Output .basis/.KTX2 files to specified directory.\n" - " -debug or -verbose: Enable codec debug print to stdout (slightly slower).\n" - " -debug_images: Enable codec debug images (much slower).\n" - " -stats: Compute and display image quality metrics (slightly to much slower).\n" - " -individual: Process input images individually and output multiple .basis/.KTX2 files (not as a texture array - this is now the default as of v1.16)\n" - "\n" - " -fastest: Set UASTC LDR 4x4 and HDR 4x4/6x6 to fastest but lowest quality encoding mode (same as -uastc_level 0 or -hdr_6x6_level 0)\n" - " -slower: Set UASTC LDR 4x4 and HDR 4x4/6x6 to slower but a higher quality encoding mode (same as -uastc_level 3 or -hdr_6x6_level 5)\n" - " -parallel: Compress multiple textures simumtanously (one per thread), instead of one at a time. Compatible with OpenCL mode. This is much faster, but in OpenCL mode the driver is pushed harder, and the CLI output will be jumbled.\n" - " -linear: Use linear colorspace metrics (instead of the default sRGB or scaled RGB for HDR), and by default linear (not sRGB) mipmap filtering.\n" - " -tex_type <2d, 2darray, 3d, video, cubemap>: Set Basis file header's texture type field. Cubemap arrays require multiples of 6 images, in X+, X-, Y+, Y-, Z+, Z- order, each image must be the same resolutions.\n" - " 2d=arbitrary 2D images, 2darray=2D array, 3D=volume texture slices, video=video frames, cubemap=array of faces. For 2darray/3d/cubemaps/video, each source image's dimensions and # of mipmap levels must be the same.\n" - " For video, the .basis file will be written with the first frame being an I-Frame, and subsequent frames being P-Frames (using conditional replenishment). Playback must always occur in order from first to last image.\n" - " -cubemap: same as -tex_type cubemap\n" - " -tex_array: Process input images as a single texture array and write a single .basis/.KTX2 file (the former default before v1.16)\n" - " -fuzz_testing: Use with -validate: Disables CRC16 validation of file contents before transcoding\n" - " -multifile_printf: printf() format strint to use to compose multiple filenames\n" - " -multifile_first: The index of the first file to process, default is 0 (must specify -multifile_printf and -multifile_num)\n" - " -multifile_num: The total number of files to process.\n" - " -opencl: Enable OpenCL usage (currently only accelerates ETC1S encoding)\n" - " -opencl_serialize: Serialize all calls to the OpenCL driver (to work around buggy drivers, only useful with -parallel)\n" - "\n" - "--- ETC1S specific options (-etc1s - the LDR/SDR default):\n" - " -q X: Set ETC1S quality level, 1-255, default is 128, lower=better compression/lower quality/faster, higher=less compression/higher quality/slower, default is 128. For even higher quality, use -max_endpoints/-max_selectors.\n" - " -comp_level X: Set ETC1S encoding speed vs. quality tradeoff. Range is 0-6, default is 1. Higher values=MUCH slower, but slightly higher quality. Higher levels intended for videos. Use -q first!\n" - " -max_endpoints X: ETC1S: Manually set the max number of color endpoint clusters from 1-16128, use instead of -q\n" - " -max_selectors X: ETC1S: Manually set the max number of color selector clusters from 1-16128, use instead of -q\n" - "\n" - "--- UASTC LDR/HDR 4x4 specific options (-uastc):\n" - " -uastc: Enable UASTC LDR 4x4 texture mode, instead of the default ETC1S mode. Significantly higher texture quality, but much larger (~8bpp) files. (Note that UASTC .basis files must be losslessly compressed by the user.)\n" - " -uastc_level: Set UASTC LDR/HDR 4x4 encoding level. LDR Range is [0,4], default is 2, higher=slower but higher quality. 0=fastest/lowest quality, 3=slowest practical option, 4=impractically slow/highest achievable quality\n" - " UASTC HDR 4x4 range is [0,4] - higher=slower but higher quality. HDR 4x4 default level=1.\n" - " -uastc_rdo_l X: Enable UASTC LDR 4x4 RDO post-processing and set UASTC LDR 4x4 RDO quality scalar (lambda) to X. Lower values=higher quality/larger LZ\n" - " compressed files, higher values=lower quality/smaller LZ compressed files. Good range to try is [.25-10].\n" - " Note: Previous versons used the -uastc_rdo_q option, which was removed because the RDO algorithm was changed.\n" - " -uastc_rdo_d X: Set UASTC LDR 4x4 RDO dictionary size in bytes. Default is 4096, max is 65536. Lower values=faster, but less compression.\n" - " -uastc_rdo_b X: Set UASTC LDR 4x4 RDO max smooth block error scale. Range is [1,300]. Default is 10.0, 1.0=disabled. Larger values suppress more artifacts (and allocate more bits) on smooth blocks.\n" - " -uastc_rdo_s X: Set UASTC LDR 4x4 RDO max smooth block standard deviation. Range is [.01,65536]. Default is 18.0. Larger values expand the range of blocks considered smooth.\n" - " -uastc_rdo_f: Don't favor simpler UASTC LDR 4x4 modes in RDO mode.\n" - " -uastc_rdo_m: Disable RDO multithreading (slightly higher compression, deterministic).\n" - "\n" - "--- UASTC HDR 4x4 specific options (-hdr or -hdr_4x4 - the HDR default):\n" - " -uastc_level X: Sets the UASTC HDR 4x4 compressor's level. Valid range is [0,4] - higher=slower but higher quality. HDR default=1.\n" - " Level 0=fastest/lowest quality, 3=highest practical setting, 4=exhaustive\n" - " -hdr_uber_mode: Allow the UASTC HDR 4x4 encoder to try varying the CEM 11 selectors more for slightly higher quality (slower). This may negatively impact BC6H quality, however.\n" - " -hdr_ultra_quant: UASTC HDR 4x4: Try to find better quantized CEM 7/11 endpoint values (slower).\n" - " -hdr_favor_astc: UASTC HDR 4x4: By default the UASTC HDR 4x4 encoder tries to strike a balance or even slightly favor BC6H quality. If this option is specified, ASTC HDR 4x4 quality is favored instead.\n" - "\n" - "--- UASTC HDR 6x6 specific options (-hdr_6x6 or -hdr_6x6i):\n" - " -lambda X: Enables rate distortion optimization (RDO). The higher this value, the lower the quality, but the smaller the file size. Try 100-20000, or higher values on some images.\n" - " -hdr_6x6_level X: Sets the codec to 6x6 HDR mode (same as -hdr_6x6) and controls encoder performance vs. max quality tradeoff. X may range from [0,12]. Default level is 2. Higher values result in better quality but slower encoding. Values above 10 are extremely slow.\n" - " -hdr_6x6i_level X: Sets the codec to 6x6 HDR intermediate mode (same as -hdr_6x6i) and controls encoder performance vs. max quality tradeoff. X may range from [0,12]. Default level is 2.\n" - " -rec_2020: The input image's gamut is Rec. 2020 vs. the default Rec. 709 - for accurate colorspace error calculations.\n" - " -hdr_6x6_jnd X, -hdr_6x6_extra_pats, -hdr_6x6_brute_force_pats, -hdr_6x6_comp_levels X Y or -hdr_6x6i_comp_levels X Y: Low-level control over the encoder's configuration.\n" - "\n" - "--- SDR/LDR->HDR upconversion options (only used when encoding to HDR formats from an LDR/SDR source image):\n" - " -hdr_ldr_no_srgb_to_linear: If specified, LDR images will NOT be converted to normalized linear light (via a sRGB->Linear conversion) during SDR->HDR upconversion before compressing as HDR.\n" - " -hdr_ldr_upconversion_nit_multiplier X: Specify how many nits (candelas per sq. meter) LDR/SDR images are converted to after converting to linear light. Default is 100 nits. Note: Previous builds used 1 nit.\n" - "\n" - "--- More options:\n" - " -test: Run an automated LDR ETC1S/UASTC encoding and transcoding test. Returns EXIT_FAILURE if any failures\n" - " -test_hdr_4x4/-test_hdr_6x6/-test_hdr_6x6i: Run automated UASTC HDR encoding and transcoding tests. Returns EXIT_FAILURE if any failures\n" - " -test_dir: Optional directory of test files. Defaults to \"../test_files\".\n" - " -y_flip: Flip input images vertically before compression\n" - " -normal_map: Tunes codec parameters for better quality on normal maps (linear colorspace metrics, linear mipmap filtering, no selector RDO, no sRGB)\n" - " -no_alpha: Always output non-alpha basis files, even if one or more inputs has alpha\n" - " -force_alpha: Always output alpha basis files, even if no inputs has alpha\n" - " -separate_rg_to_color_alpha: Separate input R and G channels to RGB and A (for tangent space XY normal maps)\n" - " -swizzle rgba: Specify swizzle for the 4 input color channels using r, g, b and a (the -separate_rg_to_color_alpha flag is equivalent to rrrg)\n" - " -renorm: Renormalize each input image before any further processing/compression\n" - " -no_multithreading: Disable multithreading\n" - " -max_threads X: Use at most X threads total when multithreading is enabled (this includes the main thread)\n" - " -no_ktx: Disable KTX writing when unpacking (faster, less output files)\n" - " -ktx_only: Only write KTX files when unpacking (faster, less output files)\n" - " -write_out: Write 3dfx OUT files when unpacking FXT1 textures\n" - " -format_only: Only unpack the specified format, by its numeric code.\n" - " -etc1_only: Only unpack to ETC1, skipping the other texture formats during -unpack\n" - " -disable_hierarchical_endpoint_codebooks: Disable hierarchical endpoint codebook usage, slower but higher quality on some compression levels\n" - " -compare_ssim: Compute and display SSIM of image comparison (slow)\n" - " -compare_plot: Display histogram plots in -compare mode\n" - " -bench: UASTC benchmark mode, for development only\n" - " -resample X Y: Resample all input textures to XxY pixels using a box filter\n" - " -resample_factor X: Resample all input textures by scale factor X using a box filter\n" - " -no_sse: Forbid all SSE instruction set usage\n" - " -validate_etc1s: Validate internal ETC1S compressor's data structures during compression (slower, intended for development).\n" - " -ktx2_animdata_duration X: Set KTX2animData duration field to integer value X (only valid/useful for -tex_type video, default is 1)\n" - " -ktx2_animdata_timescale X: Set KTX2animData timescale field to integer value X (only valid/useful for -tex_type video, default is 15)\n" - " -ktx2_animdata_loopcount X: Set KTX2animData loopcount field to integer value X (only valid/useful for -tex_type video, default is 0)\n" - " -framerate X: Set framerate in .basis header to X/frames sec.\n" - " -ktx2_no_zstandard: Don't compress UASTC texture data using Zstandard -- store it uncompressed instead.\n" - " -ktx2_zstandard_level X: Set ZStandard compression level to X (see Zstandard documentation, default level is 6)\n" - " -tonemap_dither: Dither tonemapper's 8-bit/component output by adding a small amount of white noise, only used with -tonemap mode\n" - "\n" - "--- Mipmap generation options:\n" - " -mipmap: Generate mipmaps for each source image\n" - " -mip_srgb: Convert image to linear before filtering, then back to sRGB\n" - " -mip_linear: Keep image in linear light during mipmap filtering (i.e. do not convert to/from sRGB for filtering purposes)\n" - " -mip_scale X: Set mipmap filter kernel's scale, lower=sharper, higher=more blurry, default is 1.0\n" - " -mip_filter X: Set mipmap filter kernel, default is kaiser, filters: box, tent, bell, blackman, catmullrom, mitchell, etc.\n" - " -mip_renorm: Renormalize normal map to unit length vectors after filtering\n" - " -mip_clamp: Use clamp addressing on borders, instead of wrapping\n" - " -mip_fast: Use faster mipmap generation (resample from previous mip, not always first/largest mip level). The default (as of 1/2021)\n" - " -mip_slow: Always resample each mipmap level starting from the largest mipmap. Higher quality, but slower. Opposite of -mip_fast. Was the prior default before 1/2021.\n" - " -mip_smallest X: Set smallest pixel dimension for generated mipmaps, default is 1 pixel\n" - " By default, textures will be converted from sRGB to linear light before mipmap filtering, then back to sRGB (for the RGB color channels) unless -linear is specified.\n" - " You can override this behavior with -mip_srgb/-mip_linear.\n" - "\n" - "--- ETC1S backend endpoint/selector RDO codec options:\n" - " -no_selector_rdo: Disable backend's selector rate distortion optimizations (slightly faster, less noisy output, but lower quality per output bit)\n" - " -selector_rdo_thresh X: Set selector RDO quality threshold, default is 1.25, lower is higher quality but less quality per output bit (try 1.0-3.0)\n" - " -no_endpoint_rdo: Disable backend's endpoint rate distortion optimizations (slightly faster, less noisy output, but lower quality per output bit)\n" - " -endpoint_rdo_thresh X: Set endpoint RDO quality threshold, default is 1.5, lower is higher quality but less quality per output bit (try 1.0-3.0)\n" - "\n" - "--- Set various fields in the Basis file header:\n" - " -userdata0 X: Set 32-bit userdata0 field in Basis file header to X (X is a signed 32-bit int)\n" - " -userdata1 X: Set 32-bit userdata1 field in Basis file header to X (X is a signed 32-bit int)\n" - "\n" - "--- Example LDR ETC1S/UASTC LDR 4x4 command lines:\n" - " basisu x.png : Compress sRGB image x.png to x.ktx2 using default settings (multiple filenames OK, use -tex_array if you want a tex array vs. multiple output files)\n" - " basisu -basis x.qoi : Compress sRGB image x.qoi to x.basis (supports 24-bit or 32-bit .QOI files)\n" - " basisu x.ktx2 : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" - " basisu x.basis : Unpack x.basis to PNG/KTX files (multiple filenames OK)\n" - " basisu -uastc x.png -uastc_rdo_l 2.0 -ktx2 -stats : Compress to a UASTC .KTX2 file with RDO (rate distortion optimization) to reduce .KTX2 compressed file size\n" - " basisu -file x.png -mipmap -y_flip : Compress a mipmapped x.ktx2 file from an sRGB image named x.png, Y flip each source image\n" - " basisu -validate -file x.basis : Validate x.basis (check header, check file CRC's, attempt to transcode all slices)\n" - " basisu -unpack -file x.basis : Validates, transcodes and unpacks x.basis to mipmapped .KTX and RGB/A .PNG files (transcodes to all supported GPU texture formats)\n" - " basisu -q 255 -file x.png -mipmap -debug -stats : Compress sRGB x.png to x.ktx2 at quality level 255 with compressor debug output/statistics\n" - " basisu -linear -max_endpoints 16128 -max_selectors 16128 -file x.png : Compress non-sRGB x.png to x.ktx2 using the largest supported manually specified codebook sizes\n" - " basisu -basis -comp_level 2 -max_selectors 8192 -max_endpoints 8192 -tex_type video -framerate 20 -multifile_printf \"x%02u.png\" -multifile_first 1 -multifile_num 20 : Compress a 20 sRGB source image video sequence (x01.png, x02.png, x03.png, etc.) to x01.basis\n" - "\n" - "--- Example UASTC HDR 4x4 command lines:\n" - " basisu x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 4x4 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before compression. See HDR upconversion options, above.\n" - " basisu -hdr_4x4 x.exr : Compress a HDR .EXR image to a UASTC HDR 4x4 .KTX2 file.\n" - " basisu x.hdr -uastc_level 0 : Compress a HDR .hdr image to a UASTC HDR 4x4 .KTX2 file, fastest encoding but lowest quality\n" - " basisu -hdr x.png : Compress a LDR .PNG image to UASTC HDR 4x4 (image is converted from sRGB to linear light first, use -hdr_ldr_no_srgb_to_linear to disable)\n" - " basisu x.hdr -uastc_level 3 : Compress a HDR .hdr image to UASTC HDR 4x4 at higher quality (-uastc_level 4 is highest quality, but very slow encoding)\n" - " basisu x.hdr -uastc_level 3 -mipmap -basis -stats -debug -debug_images : Compress a HDR .hdr image to UASTC HDR 4x4, .basis output file, at higher quality, generate mipmaps, output statistics and debug information, and write tone mapped debug images\n" - " basisu x.hdr -stats -hdr_favor_astc -hdr_uber_mode -uastc_level 4 : Highest achievable ASTC HDR 4x4 quality (very slow encoding, BC6H quality is traded off)\n" - "\n--- Example UASTC HDR 6x6 command lines:\n" - " basisu -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 6x6 .KTX2 file. LDR/SDR images will be upconverted to linear light HDR before compression. See HDR upconversion options, above.\n" - " basisu -lambda 1000 -hdr_6x6 x.exr : Compress a HDR .EXR (or .HDR) image to a UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda level 1000.\n" - " basisu -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed intermediate format UASTC HDR 6x6 .KTX2 file.\n" - " basisu -lambda 1000 -hdr_6x6i x.exr : Compress a HDR .EXR image to a compressed intermediate format UASTC HDR 6x6 .KTX2 file with rate-distortion optimization (RDO), at lambda level 1000.\n" - "\n" - "--- Video notes: For video use, it's recommended to encode on a machine with many cores. Use -comp_level 2 or higher for better codebook\n" - "generation, specify very large codebooks using -max_endpoints and -max_selectors, and reduce the default endpoint RDO threshold\n" - "(-endpoint_rdo_thresh) to around 1.25. Videos may have mipmaps and alpha channels. Videos must always be played back by the transcoder\n" - "in first to last image order.\n" - "Video files currently use I-Frames on the first image, and P-Frames using conditional replenishment on subsequent frames.\n" - "\nETC1S Compression level (-comp_level X) details. This controls the ETC1S speed vs. quality trandeoff. (Use -q to control the quality vs. compressed size tradeoff.):\n" - " Level 0: Fastest, but has marginal quality and can be brittle on complex images. Avg. Y dB: 35.45\n" - " Level 1: Hierarchical codebook searching, faster ETC1S encoding. 36.87 dB, ~1.4x slower vs. level 0. (This is the default setting.)\n" - " Level 2: Use this or higher for video. Hierarchical codebook searching. 36.87 dB, ~1.4x slower vs. level 0. (This is the v1.12's default setting.)\n" - " Level 3: Full codebook searching. 37.13 dB, ~1.8x slower vs. level 0. (Equivalent the the initial release's default settings.)\n" - " Level 4: Hierarchical codebook searching, codebook k-means iterations. 37.15 dB, ~4x slower vs. level 0\n" - " Level 5: Full codebook searching, codebook k-means iterations. 37.41 dB, ~5.5x slower vs. level 0.\n" - " Level 6: Full codebook searching, twice as many codebook k-means iterations, best ETC1 endpoint opt. 37.43 dB, ~12x slower vs. level 0\n" + puts( +#include "basisu_tool_help.h" ); } @@ -327,7 +155,7 @@ static bool load_listing_file(const std::string &f, basisu::vector { if (read_filename[0] == ' ') read_filename.erase(0, 1); - else + else break; } @@ -336,7 +164,7 @@ static bool load_listing_file(const std::string &f, basisu::vector const char c = read_filename.back(); if ((c == ' ') || (c == '\n') || (c == '\r')) read_filename.erase(read_filename.size() - 1, 1); - else + else break; } @@ -360,9 +188,347 @@ class command_line_params #define REMAINING_ARGS_CHECK(n) if (num_remaining_args < (n)) { error_printf("Error: Expected %u values to follow %s!\n", n, pArg); return false; } + bool check_for_general_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) + { + BASISU_NOTE_UNUSED(arg_v); + BASISU_NOTE_UNUSED(arg_index); + BASISU_NOTE_UNUSED(num_remaining_args); + BASISU_NOTE_UNUSED(arg_count); + + if (strcasecmp(pArg, "-wasi_threads") == 0) + { + REMAINING_ARGS_CHECK(1); + int num_threads = atoi(arg_v[arg_index + 1]); + if ((num_threads < 0) || (num_threads > 256)) + { + error_printf("Invalid number of threads\n"); + exit(EXIT_FAILURE); + } + set_num_wasi_threads(num_threads); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-higher_quality_transcoding") == 0) + { + m_higher_quality_transcoding = true; + return true; + } + else if (strcasecmp(pArg, "-no_fast_xuastc_ldr_bc7_transcoding") == 0) + { + m_xuastc_ldr_disable_bc7_transcoding = true; + return true; + } + else if (strcasecmp(pArg, "-fast_xuastc_ldr_bc7_transcoding") == 0) + { + m_xuastc_ldr_disable_bc7_transcoding = false; + return true; + } + else if (strcasecmp(pArg, "-no_etc1s_chroma_filtering") == 0) + { + m_no_etc1s_transcoding_chroma_filtering = true; + return true; + } + else if (strcasecmp(pArg, "-force_deblocking") == 0) + { + m_force_deblocking = true; + return true; + } + else if ((strcasecmp(pArg, "-disable_deblocking") == 0) || (strcasecmp(pArg, "-no_deblocking") == 0)) + { + m_disable_deblocking = true; + return true; + } + else if (strcasecmp(pArg, "-stronger_deblocking") == 0) + { + m_stronger_deblocking = true; + return true; + } + + return false; + } + + bool check_for_xuastc_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) + { + // New unified -quality level which works across all codecs + if (strcasecmp(pArg, "-quality") == 0) + { + REMAINING_ARGS_CHECK(1); + m_quality_level = clamp(atoi(arg_v[arg_index + 1]), 0, 100); + arg_count++; + return true; + } + // New unified -effort level, which works across all codecs + else if (strcasecmp(pArg, "-effort") == 0) + { + REMAINING_ARGS_CHECK(1); + m_effort_level = clamp(atoi(arg_v[arg_index + 1]), 0, 10); + //m_comp_params.m_xuastc_ldr_effort_level = atoi(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-xuastc_blurring") == 0) // experimental, not recommended, very slow + { + m_comp_params.m_xuastc_ldr_blurring = true; + return true; + } + else if (strcasecmp(pArg, "-weights") == 0) + { + REMAINING_ARGS_CHECK(4); + m_comp_params.m_xuastc_ldr_channel_weights[0] = (uint32_t)clamp((float)atof(arg_v[arg_index + 1]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[1] = (uint32_t)clamp((float)atof(arg_v[arg_index + 2]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[2] = (uint32_t)clamp((float)atof(arg_v[arg_index + 3]), 0.0f, 1024.0f); + m_comp_params.m_xuastc_ldr_channel_weights[3] = (uint32_t)clamp((float)atof(arg_v[arg_index + 4]), 0.0f, 1024.0f); + arg_count += 4; + return true; + } + else if (strcasecmp(pArg, "-ls_min_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_min_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_min_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_min_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_edge_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_edge_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-ls_thresh_edge_alpha_psnr") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_ls_thresh_edge_alpha_psnr = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-xuastc_arith") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith; + return true; + } + else if (strcasecmp(pArg, "-xuastc_zstd") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd; + return true; + } + else if (strcasecmp(pArg, "-xuastc_hybrid") == 0) + { + m_comp_params.m_xuastc_ldr_syntax = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd; + return true; + } + else if (strcasecmp(pArg, "-xy") == 0) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = true; + return true; + } + else if (strcasecmp(pArg, "-xyd") == 0) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = false; + return true; + } + else if (strcasecmp(pArg, "-xs") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_subsets = true; + return true; + } + else if (strcasecmp(pArg, "-xsu") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_subsets = false; + return true; + } + else if (strcasecmp(pArg, "-xp") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_rgb_dual_plane = true; + return true; + } + else if (strcasecmp(pArg, "-xpu") == 0) + { + m_comp_params.m_xuastc_ldr_force_disable_rgb_dual_plane = false; + return true; + } + else if (strcasecmp(pArg, "-ts") == 0) + { + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; + return true; + } + else if (strcasecmp(pArg, "-tl") == 0) + { + m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; + return true; + } + // Supercompressed XUASTC LDR 4x4-12x12 + else if ((strcasecmp(pArg, "-ldr_4x4i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_4x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x4i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_5x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_5x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_5x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_5x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_6x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_6x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_6x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_6x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x5i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x6i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x8i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_8x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_8x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x8i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x10i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_10x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_10x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x10i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_12x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_12x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x12i") == 0) || (strcasecmp(pArg, "-xuastc_ldr_12x12") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cXUASTC_LDR_12x12); + return true; + } + // Plain ASTC LDR 4x4-12x12 + else if ((strcasecmp(pArg, "-ldr_4x4") == 0) || (strcasecmp(pArg, "-astc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_4x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x4") == 0) || (strcasecmp(pArg, "-astc_ldr_5x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_5x4); + return true; + } + else if ((strcasecmp(pArg, "-ldr_5x5") == 0) || (strcasecmp(pArg, "-astc_ldr_5x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_5x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x5") == 0) || (strcasecmp(pArg, "-astc_ldr_6x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_6x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_6x6") == 0) || (strcasecmp(pArg, "-astc_ldr_6x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_6x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x5") == 0) || (strcasecmp(pArg, "-astc_ldr_8x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x6") == 0) || (strcasecmp(pArg, "-astc_ldr_8x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x5") == 0) || (strcasecmp(pArg, "-astc_ldr_10x5") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x5); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x6") == 0) || (strcasecmp(pArg, "-astc_ldr_10x6") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x6); + return true; + } + else if ((strcasecmp(pArg, "-ldr_8x8") == 0) || (strcasecmp(pArg, "-astc_ldr_8x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_8x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x8") == 0) || (strcasecmp(pArg, "-astc_ldr_10x8") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x8); + return true; + } + else if ((strcasecmp(pArg, "-ldr_10x10") == 0) || (strcasecmp(pArg, "-astc_ldr_10x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_10x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x10") == 0) || (strcasecmp(pArg, "-astc_ldr_12x10") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_12x10); + return true; + } + else if ((strcasecmp(pArg, "-ldr_12x12") == 0) || (strcasecmp(pArg, "-astc_ldr_12x12") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_LDR_12x12); + return true; + } + + return false; + } + bool check_for_hdr_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) { - if ((strcasecmp(pArg, "-hdr") == 0) || (strcasecmp(pArg, "-hdr_4x4") == 0)) + if ((strcasecmp(pArg, "-hdr") == 0) || (strcasecmp(pArg, "-hdr_4x4") == 0) || (strcasecmp(pArg, "-uastc_hdr_4x4") == 0)) { m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_4x4); return true; @@ -372,16 +538,16 @@ class command_line_params m_comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = true; return true; } - else if (strcasecmp(pArg, "-hdr_6x6") == 0) + else if ((strcasecmp(pArg, "-hdr_6x6") == 0) || (strcasecmp(pArg, "-astc_hdr_6x6") == 0)) { // max quality (if -lambda=0) or RDO UASTC HDR 6x6 m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); return true; } - else if (strcasecmp(pArg, "-hdr_6x6i") == 0) + else if ((strcasecmp(pArg, "-hdr_6x6i") == 0) || (strcasecmp(pArg, "-uastc_hdr_6x6") == 0)) { // intermediate format UASTC HDR 6x6 - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); return true; } else if (strcasecmp(pArg, "-lambda") == 0) @@ -401,6 +567,8 @@ class command_line_params m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); m_comp_params.m_rdo_uastc_ldr_4x4 = true; + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -418,6 +586,9 @@ class command_line_params const int level = atoi(arg_v[arg_index + 1]); m_comp_params.m_astc_hdr_6x6_options.set_user_level(level); m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); + + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -426,7 +597,10 @@ class command_line_params REMAINING_ARGS_CHECK(1); const int level = atoi(arg_v[arg_index + 1]); m_comp_params.m_astc_hdr_6x6_options.set_user_level(level); - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + m_used_old_style_codec_config_param = true; + arg_count++; return true; } @@ -444,16 +618,19 @@ class command_line_params { REMAINING_ARGS_CHECK(2); + // Intended for low-level/development/testing const int lo_level = clamp(atoi(arg_v[arg_index + 1]), 0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_COMP_LEVEL); const int hi_level = clamp(atoi(arg_v[arg_index + 2]), 0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_COMP_LEVEL); m_comp_params.m_astc_hdr_6x6_options.m_master_comp_level = minimum(lo_level, hi_level); m_comp_params.m_astc_hdr_6x6_options.m_highest_comp_level = maximum(lo_level, hi_level); - + if (strcasecmp(pArg, "-hdr_6x6_comp_levels") == 0) m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6); else - m_comp_params.set_format_mode(basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + m_used_old_style_codec_config_param = true; arg_count += 2; return true; @@ -514,6 +691,179 @@ class command_line_params return false; } + // ETC1S or UASTC LDR 4x4 specific options + bool check_for_etc1s_or_uastc_options(const char** arg_v, const char* pArg, int arg_index, const int num_remaining_args, int& arg_count) + { + if (strcasecmp(pArg, "-etc1s") == 0) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cETC1S); + return true; + } + else if ((strcasecmp(pArg, "-uastc") == 0) || (strcasecmp(pArg, "-uastc_ldr") == 0) || (strcasecmp(pArg, "-uastc_ldr_4x4") == 0)) + { + m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC_LDR_4x4); + return true; + } + else if (strcasecmp(pArg, "-uastc_level") == 0) + { + REMAINING_ARGS_CHECK(1); + + int uastc_level = atoi(arg_v[arg_index + 1]); + + uastc_level = clamp(uastc_level, 0, TOTAL_PACK_UASTC_LEVELS - 1); + + static_assert(TOTAL_PACK_UASTC_LEVELS == 5, "TOTAL_PACK_UASTC_LEVELS==5"); + static const uint32_t s_level_flags[TOTAL_PACK_UASTC_LEVELS] = { cPackUASTCLevelFastest, cPackUASTCLevelFaster, cPackUASTCLevelDefault, cPackUASTCLevelSlower, cPackUASTCLevelVerySlow }; + + m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; + m_comp_params.m_pack_uastc_ldr_4x4_flags |= s_level_flags[uastc_level]; + + m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_level); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_l") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); + m_comp_params.m_rdo_uastc_ldr_4x4 = true; + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_d") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_dict_size = atoi(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_b") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_s") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_f") == 0) + { + m_comp_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode = false; + return true; + } + else if (strcasecmp(pArg, "-uastc_rdo_m") == 0) + { + m_comp_params.m_rdo_uastc_ldr_4x4_multithreading = false; + return true; + } + else if (strcasecmp(pArg, "-validate_etc1s") == 0) + { + m_comp_params.m_validate_etc1s = true; + return true; + } + else if (strcasecmp(pArg, "-comp_level") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_compression_level = atoi(arg_v[arg_index + 1]); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-max_endpoints") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_max_endpoint_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_ENDPOINT_CLUSTERS); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-max_selectors") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_etc1s_max_selector_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_SELECTOR_CLUSTERS); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } +#if 0 + else if (strcasecmp(pArg, "-gen_global_codebooks") == 0) + { + // TODO + } +#endif + else if (strcasecmp(pArg, "-use_global_codebooks") == 0) + { + REMAINING_ARGS_CHECK(1); + m_etc1s_use_global_codebooks_file = arg_v[arg_index + 1]; + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-etc1_only") == 0) + { + m_etc1_only = true; + m_unpack_format_only = (int)basist::transcoder_texture_format::cTFETC1_RGB; + return true; + } + else if (strcasecmp(pArg, "-disable_hierarchical_endpoint_codebooks") == 0) + { + m_comp_params.m_disable_hierarchical_endpoint_codebooks = true; + return true; + } + else if (strcasecmp(pArg, "-q") == 0) // old-style -q, prefer -quality instead + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_quality_level = clamp(atoi(arg_v[arg_index + 1]), BASISU_QUALITY_MIN, BASISU_QUALITY_MAX); + + m_used_old_style_codec_config_param = true; + + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-no_selector_rdo") == 0) + { + m_comp_params.m_no_selector_rdo = true; + return true; + } + else if (strcasecmp(pArg, "-selector_rdo_thresh") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_selector_rdo_thresh = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + else if (strcasecmp(pArg, "-no_endpoint_rdo") == 0) + { + m_comp_params.m_no_endpoint_rdo = true; + return true; + } + else if (strcasecmp(pArg, "-endpoint_rdo_thresh") == 0) + { + REMAINING_ARGS_CHECK(1); + m_comp_params.m_endpoint_rdo_thresh = (float)atof(arg_v[arg_index + 1]); + arg_count++; + return true; + } + + return false; + } + public: command_line_params() : m_mode(cDefault), @@ -523,7 +873,7 @@ public: m_ktx2_animdata_duration(1), m_ktx2_animdata_timescale(15), m_ktx2_animdata_loopcount(0), - m_format_only(-1), + m_unpack_format_only(-1), m_multifile_first(0), m_multifile_num(0), m_max_threads(1024), // surely this is high enough @@ -536,12 +886,26 @@ public: m_compare_ssim(false), m_compare_plot(false), m_parallel_compression(false), - m_tonemap_dither_flag(false) + m_tonemap_dither_flag(false), + m_xuastc_ldr_disable_bc7_transcoding(false), + m_no_etc1s_transcoding_chroma_filtering(false), + m_higher_quality_transcoding(false), + m_force_deblocking(false), + m_disable_deblocking(false), + m_stronger_deblocking(false), + m_effort_level(-1), + m_quality_level(-1), + m_used_old_style_codec_config_param(false) { - m_comp_params.m_compression_level = basisu::maximum(0, BASISU_DEFAULT_COMPRESSION_LEVEL - 1); - + // This command line tool defaults to ETC1S level 1, not 2 which is the API default (for backwards compat). + m_comp_params.m_etc1s_compression_level = maximum((int)BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL - 1, 0); + m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_hdr_4x4_codec_options::cDefaultLevel); + // Default to sRGB colorspace metrics/transfer functions (independent of the code defaults). + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; + m_test_file_dir = "../test_files"; } @@ -559,8 +923,20 @@ public: print_usage(); exit(EXIT_SUCCESS); } - - if (strcasecmp(pArg, "-ktx2") == 0) + + if (check_for_etc1s_or_uastc_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_hdr_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_xuastc_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (check_for_general_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) + { + } + else if (strcasecmp(pArg, "-ktx2") == 0) { m_ktx2_mode = true; } @@ -626,6 +1002,8 @@ public: m_mode = cCompSize; else if ((strcasecmp(pArg, "-test") == 0) || (strcasecmp(pArg, "-test_ldr") == 0)) m_mode = cTestLDR; + else if ((strcasecmp(pArg, "-test_xuastc") == 0) || (strcasecmp(pArg, "-test_xuastc_ldr") == 0)) + m_mode = cTestXUASTCLDR; else if (strcasecmp(pArg, "-test_hdr_4x4") == 0) m_mode = cTestHDR_4x4; else if (strcasecmp(pArg, "-test_hdr_6x6") == 0) @@ -646,7 +1024,7 @@ public: g_cpu_supports_sse41 = false; #endif } - else if (strcasecmp(pArg, "-no_status_output") == 0) + else if ((strcasecmp(pArg, "-no_status_output") == 0) || (strcasecmp(pArg, "-quiet") == 0)) { m_comp_params.m_status_output = false; } @@ -680,50 +1058,6 @@ public: m_multifile_num = atoi(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-uastc") == 0) - { - m_comp_params.set_format_mode(basist::basis_tex_format::cUASTC4x4); - } - else if (strcasecmp(pArg, "-etc1s") == 0) - { - m_comp_params.set_format_mode(basist::basis_tex_format::cETC1S); - } - else if (strcasecmp(pArg, "-fastest") == 0) - { - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= cPackUASTCLevelFastest; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(0); - - m_comp_params.m_astc_hdr_6x6_options.set_user_level(0); - } - else if (strcasecmp(pArg, "-slower") == 0) - { - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= cPackUASTCLevelSlower; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(3); - - m_comp_params.m_astc_hdr_6x6_options.set_user_level(5); - } - else if (strcasecmp(pArg, "-uastc_level") == 0) - { - REMAINING_ARGS_CHECK(1); - - int uastc_level = atoi(arg_v[arg_index + 1]); - - uastc_level = clamp(uastc_level, 0, TOTAL_PACK_UASTC_LEVELS - 1); - - static_assert(TOTAL_PACK_UASTC_LEVELS == 5, "TOTAL_PACK_UASTC_LEVELS==5"); - static const uint32_t s_level_flags[TOTAL_PACK_UASTC_LEVELS] = { cPackUASTCLevelFastest, cPackUASTCLevelFaster, cPackUASTCLevelDefault, cPackUASTCLevelSlower, cPackUASTCLevelVerySlow }; - - m_comp_params.m_pack_uastc_ldr_4x4_flags &= ~cPackUASTCLevelMask; - m_comp_params.m_pack_uastc_ldr_4x4_flags |= s_level_flags[uastc_level]; - - m_comp_params.m_uastc_hdr_4x4_options.set_quality_level(uastc_level); - - arg_count++; - } else if (strcasecmp(pArg, "-resample") == 0) { REMAINING_ARGS_CHECK(2); @@ -737,44 +1071,15 @@ public: m_comp_params.m_resample_factor = (float)atof(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-uastc_rdo_l") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = (float)atof(arg_v[arg_index + 1]); - m_comp_params.m_rdo_uastc_ldr_4x4 = true; - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_d") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_dict_size = atoi(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_b") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_s") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-uastc_rdo_f") == 0) - m_comp_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode = false; - else if (strcasecmp(pArg, "-uastc_rdo_m") == 0) - m_comp_params.m_rdo_uastc_ldr_4x4_multithreading = false; else if (strcasecmp(pArg, "-linear") == 0) - m_comp_params.m_perceptual = false; - else if (strcasecmp(pArg, "-srgb") == 0) - m_comp_params.m_perceptual = true; - else if (strcasecmp(pArg, "-q") == 0) { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_quality_level = clamp(atoi(arg_v[arg_index + 1]), BASISU_QUALITY_MIN, BASISU_QUALITY_MAX); - arg_count++; + m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; + } + else if (strcasecmp(pArg, "-srgb") == 0) + { + m_comp_params.m_perceptual = true; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = true; } else if (strcasecmp(pArg, "-output_file") == 0) { @@ -793,10 +1098,6 @@ public: m_comp_params.m_debug = true; enable_debug_printf(true); } - else if (strcasecmp(pArg, "-validate_etc1s") == 0) - { - m_comp_params.m_validate_etc1s = true; - } else if (strcasecmp(pArg, "-validate_output") == 0) { m_comp_params.m_validate_output_data = true; @@ -805,39 +1106,12 @@ public: m_comp_params.m_debug_images = true; else if (strcasecmp(pArg, "-stats") == 0) m_comp_params.m_compute_stats = true; - else if (strcasecmp(pArg, "-gen_global_codebooks") == 0) - { - // TODO - } - else if (strcasecmp(pArg, "-use_global_codebooks") == 0) - { - REMAINING_ARGS_CHECK(1); - m_etc1s_use_global_codebooks_file = arg_v[arg_index + 1]; - arg_count++; - } - else if (strcasecmp(pArg, "-comp_level") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_compression_level = atoi(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-max_endpoints") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_max_endpoint_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_ENDPOINT_CLUSTERS); - arg_count++; - } - else if (strcasecmp(pArg, "-max_selectors") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_etc1s_max_selector_clusters = clamp(atoi(arg_v[arg_index + 1]), 1, BASISU_MAX_SELECTOR_CLUSTERS); - arg_count++; - } else if (strcasecmp(pArg, "-y_flip") == 0) m_comp_params.m_y_flip = true; else if (strcasecmp(pArg, "-normal_map") == 0) { m_comp_params.m_perceptual = false; + m_comp_params.m_ktx2_and_basis_srgb_transfer_function = false; m_comp_params.m_mip_srgb = false; m_comp_params.m_no_selector_rdo = true; m_comp_params.m_no_endpoint_rdo = true; @@ -894,7 +1168,7 @@ public: else if (strcasecmp(pArg, "-max_threads") == 0) { REMAINING_ARGS_CHECK(1); - m_max_threads = atoi(arg_v[arg_index + 1]); + m_max_threads = maximum(1, atoi(arg_v[arg_index + 1])); arg_count++; } else if (strcasecmp(pArg, "-mipmap") == 0) @@ -910,19 +1184,9 @@ public: else if (strcasecmp(pArg, "-format_only") == 0) { REMAINING_ARGS_CHECK(1); - m_format_only = atoi(arg_v[arg_index + 1]); + m_unpack_format_only = atoi(arg_v[arg_index + 1]); arg_count++; } - else if (strcasecmp(pArg, "-etc1_only") == 0) - { - m_etc1_only = true; - m_format_only = (int)basist::transcoder_texture_format::cTFETC1_RGB; - } - else if (strcasecmp(pArg, "-disable_hierarchical_endpoint_codebooks") == 0) - m_comp_params.m_disable_hierarchical_endpoint_codebooks = true; - else if (check_for_hdr_options(arg_v, pArg, arg_index, num_remaining_args, arg_count)) - { - } else if (strcasecmp(pArg, "-opencl") == 0) { m_comp_params.m_use_opencl = true; @@ -940,7 +1204,7 @@ public: { REMAINING_ARGS_CHECK(1); m_comp_params.m_mip_filter = arg_v[arg_index + 1]; - // TODO: Check filter + // TODO: Check filter arg_count++; } else if (strcasecmp(pArg, "-mip_renorm") == 0) @@ -961,22 +1225,6 @@ public: m_comp_params.m_mip_srgb = true; else if (strcasecmp(pArg, "-mip_linear") == 0) m_comp_params.m_mip_srgb = false; - else if (strcasecmp(pArg, "-no_selector_rdo") == 0) - m_comp_params.m_no_selector_rdo = true; - else if (strcasecmp(pArg, "-selector_rdo_thresh") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_selector_rdo_thresh = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } - else if (strcasecmp(pArg, "-no_endpoint_rdo") == 0) - m_comp_params.m_no_endpoint_rdo = true; - else if (strcasecmp(pArg, "-endpoint_rdo_thresh") == 0) - { - REMAINING_ARGS_CHECK(1); - m_comp_params.m_endpoint_rdo_thresh = (float)atof(arg_v[arg_index + 1]); - arg_count++; - } else if (strcasecmp(pArg, "-userdata0") == 0) { REMAINING_ARGS_CHECK(1); @@ -1068,20 +1316,28 @@ public: arg_index += arg_count; assert(arg_index <= arg_c); } - - if (m_comp_params.m_etc1s_quality_level != -1) + + if (m_comp_params.m_quality_level != -1) // old-style -q X option { m_comp_params.m_etc1s_max_endpoint_clusters = 0; m_comp_params.m_etc1s_max_selector_clusters = 0; + + // -q also controls XUASTC LDR weight grid DCT quality level + m_comp_params.m_xuastc_ldr_use_dct = true; + + // Automatically enable lossy XUASTC supercompression if DCT is enabled. + if (!m_comp_params.m_xuastc_ldr_use_lossy_supercompression.was_changed()) + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = true; } else if ((!m_comp_params.m_etc1s_max_endpoint_clusters) || (!m_comp_params.m_etc1s_max_selector_clusters)) { m_comp_params.m_etc1s_max_endpoint_clusters = 0; m_comp_params.m_etc1s_max_selector_clusters = 0; - m_comp_params.m_etc1s_quality_level = 128; + m_comp_params.m_quality_level = 128; } - + + // Ensure mip_srgb is set to match the perceptual flag if the user didn't explicitly set it. if (!m_comp_params.m_mip_srgb.was_changed()) { // They didn't specify what colorspace to do mipmap filtering in, so choose sRGB if they've specified that the texture is sRGB. @@ -1091,6 +1347,30 @@ public: m_comp_params.m_mip_srgb = false; } + // Handle new-style unified effort and quality levels across all codecs. + // We have so many codecs now that it's necessary to unify the primary quality/effort controls otherwise it's too confusing. + // If they've specified either -effort or -quality, assume they want the new unified API. + // If they haven't specified either, they get the old parameters/options. + if ((m_effort_level != -1) || (m_quality_level != -1)) + { + if (m_used_old_style_codec_config_param) + { + fmt_printf("WARNING: Mixing old and new-style (-effort and/or -quality) codec configuration parameters.\nNew-style parameters may overwrite your old-style codec configuration settings.\nPrefer using -effort X and -quality X."); + } + + const bool lossy_supercompression_changed = m_comp_params.m_xuastc_ldr_use_lossy_supercompression.was_changed(); + const bool lossy_supercompression_value = m_comp_params.m_xuastc_ldr_use_lossy_supercompression; + + // Set the new-style effort/quality level, but importantly don't override any settings already changed if they haven't explictly specified -effort or -quality. + m_comp_params.set_format_mode_and_quality_effort(m_comp_params.get_format_mode(), m_quality_level, m_effort_level, false); + + // Allow the user to override the lossy supercompression setting, independent of the quality/effort levels. + if (lossy_supercompression_changed) + { + m_comp_params.m_xuastc_ldr_use_lossy_supercompression = lossy_supercompression_value; + } + } + return true; } @@ -1121,28 +1401,28 @@ public: new_input_alpha_filenames.push_back(m_input_alpha_filenames[i]); } new_input_alpha_filenames.swap(m_input_alpha_filenames); - + return true; } basis_compressor_params m_comp_params; - + tool_mode m_mode; - + bool m_ktx2_mode; bool m_ktx2_zstandard; int m_ktx2_zstandard_level; uint32_t m_ktx2_animdata_duration; uint32_t m_ktx2_animdata_timescale; uint32_t m_ktx2_animdata_loopcount; - + basisu::vector m_input_filenames; basisu::vector m_input_alpha_filenames; std::string m_output_filename; std::string m_output_path; - int m_format_only; + int m_unpack_format_only; std::string m_multifile_printf; uint32_t m_multifile_first; @@ -1153,9 +1433,9 @@ public: std::string m_etc1s_use_global_codebooks_file; std::string m_test_file_dir; - + uint32_t m_max_threads; - + bool m_individual; bool m_no_ktx; bool m_ktx_only; @@ -1166,19 +1446,29 @@ public: bool m_compare_plot; bool m_parallel_compression; bool m_tonemap_dither_flag; + bool m_xuastc_ldr_disable_bc7_transcoding; + bool m_no_etc1s_transcoding_chroma_filtering; + bool m_higher_quality_transcoding; + bool m_force_deblocking; + bool m_disable_deblocking; + bool m_stronger_deblocking; + + int m_effort_level; + int m_quality_level; + bool m_used_old_style_codec_config_param; // true if the user has specified low-level or old-style codec configuration parameters }; static bool expand_multifile(command_line_params &opts) { if (!opts.m_multifile_printf.size()) return true; - + if (!opts.m_multifile_num) { error_printf("-multifile_printf specified, but not -multifile_num\n"); return false; } - + std::string fmt(opts.m_multifile_printf); // Workaround for MSVC debugger issues. Questionable to leave in here. size_t x = fmt.find_first_of('!'); @@ -1190,15 +1480,15 @@ static bool expand_multifile(command_line_params &opts) error_printf("Must include C-style printf() format character '%%' in -multifile_printf string\n"); return false; } - + for (uint32_t i = opts.m_multifile_first; i < opts.m_multifile_first + opts.m_multifile_num; i++) { char buf[1024]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(buf, sizeof(buf), fmt.c_str(), i); #else snprintf(buf, sizeof(buf), fmt.c_str(), i); -#endif +#endif if (buf[0]) opts.m_input_filenames.push_back(buf); @@ -1209,8 +1499,8 @@ static bool expand_multifile(command_line_params &opts) struct basis_data { - basis_data() : - m_transcoder() + basis_data() : + m_transcoder() { } uint8_vec m_file_data; @@ -1227,7 +1517,7 @@ static basis_data *load_basis_file(const char *pInput_filename, bool force_etc1s delete p; return nullptr; } - printf("Input file \"%s\"\n", pInput_filename); + printf("\nInput file \"%s\"\n", pInput_filename); if (!basis_data.size()) { error_printf("File is empty!\n"); @@ -1258,6 +1548,24 @@ static basis_data *load_basis_file(const char *pInput_filename, bool force_etc1s return p; } +static uint32_t get_transcode_flags_from_options(const command_line_params& opts) +{ + uint32_t transcode_flags = opts.m_higher_quality_transcoding ? basist::cDecodeFlagsHighQuality : 0; + + if (opts.m_disable_deblocking) + transcode_flags |= basist::cDecodeFlagsNoDeblockFiltering; + else if (opts.m_force_deblocking) + transcode_flags |= basist::cDecodeFlagsForceDeblockFiltering; + if (opts.m_stronger_deblocking) + transcode_flags |= basist::cDecodeFlagsStrongerDeblockFiltering; + if (opts.m_no_etc1s_transcoding_chroma_filtering) + transcode_flags |= basist::cDecodeFlagsNoETC1SChromaFiltering; + if (opts.m_xuastc_ldr_disable_bc7_transcoding) + transcode_flags |= basist::cDecodeFlagXUASTCLDRDisableFastBC7Transcoding; + + return transcode_flags; +} + static bool compress_mode(command_line_params &opts) { uint32_t num_threads = 1; @@ -1265,17 +1573,18 @@ static bool compress_mode(command_line_params &opts) if (opts.m_comp_params.m_multithreading) { // We use std::thread::hardware_concurrency() as a hint to determine the default # of threads to put into a pool. - num_threads = std::thread::hardware_concurrency(); + num_threads = get_num_hardware_threads(); if (num_threads < 1) num_threads = 1; if (num_threads > opts.m_max_threads) num_threads = opts.m_max_threads; } + // num_threads is the total thread pool size, *including* the calling thread. So 1=no extra threads. job_pool compressor_jpool(opts.m_parallel_compression ? 1 : num_threads); if (!opts.m_parallel_compression) opts.m_comp_params.m_pJob_pool = &compressor_jpool; - + if (!expand_multifile(opts)) { error_printf("-multifile expansion failed!\n"); @@ -1287,7 +1596,7 @@ static bool compress_mode(command_line_params &opts) error_printf("No input files to process!\n"); return false; } - + basis_data* pGlobal_codebook_data = nullptr; if (opts.m_etc1s_use_global_codebooks_file.size()) { @@ -1297,7 +1606,7 @@ static bool compress_mode(command_line_params &opts) printf("Loaded global codebooks from .basis file \"%s\"\n", opts.m_etc1s_use_global_codebooks_file.c_str()); } - + basis_compressor_params ¶ms = opts.m_comp_params; if (opts.m_ktx2_mode) @@ -1307,9 +1616,7 @@ static bool compress_mode(command_line_params &opts) params.m_ktx2_uastc_supercompression = basist::KTX2_SS_ZSTANDARD; else params.m_ktx2_uastc_supercompression = basist::KTX2_SS_NONE; - - params.m_ktx2_srgb_transfer_func = opts.m_comp_params.m_perceptual; - + if (params.m_tex_type == basist::basis_texture_type::cBASISTexTypeVideoFrames) { // Create KTXanimData key value entry @@ -1319,7 +1626,7 @@ static bool compress_mode(command_line_params &opts) const char* pAD = "KTXanimData"; kv.m_key.resize(strlen(pAD) + 1); strcpy((char*)kv.m_key.data(), pAD); - + basist::ktx2_animdata ad; ad.m_duration = opts.m_ktx2_animdata_duration; ad.m_timescale = opts.m_ktx2_animdata_timescale; @@ -1330,15 +1637,18 @@ static bool compress_mode(command_line_params &opts) params.m_ktx2_key_values.push_back(kv); } - + // TODO- expose this to command line. params.m_ktx2_zstd_supercompression_level = opts.m_ktx2_zstandard_level; } params.m_read_source_images = true; params.m_write_output_basis_or_ktx2_files = true; - params.m_pGlobal_codebooks = pGlobal_codebook_data ? &pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder() : nullptr; - + params.m_pGlobal_codebooks = pGlobal_codebook_data ? &pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder() : nullptr; + + // Get the transcode/decode flags used when validating the output by calling the transcoder from the options. + params.m_transcode_flags = get_transcode_flags_from_options(opts); + FILE *pCSV_file = nullptr; if (opts.m_csv_file.size()) { @@ -1354,7 +1664,7 @@ static bool compress_mode(command_line_params &opts) } printf("Processing %u total file(s)\n", (uint32_t)opts.m_input_filenames.size()); - + interval_timer all_tm; all_tm.start(); @@ -1403,7 +1713,7 @@ static bool compress_mode(command_line_params &opts) params.m_source_filenames = opts.m_input_filenames; params.m_source_alpha_filenames = opts.m_input_alpha_filenames; } - + if (opts.m_output_filename.size()) params.m_out_filename = opts.m_output_filename; else @@ -1459,9 +1769,11 @@ static bool compress_mode(command_line_params &opts) if (params.m_status_output) { - printf("Compression succeeded to file \"%s\" size %zu bytes in %3.3f secs\n", params.m_out_filename.c_str(), - opts.m_ktx2_mode ? c.get_output_ktx2_file().size() : c.get_output_basis_file().size(), - tm.get_elapsed_secs()); + fmt_printf("Compression succeeded to file \"{}\" size {} bytes in {3.3} secs, {3.3} bits/texel\n", + params.m_out_filename.c_str(), + opts.m_ktx2_mode ? (uint64_t)c.get_output_ktx2_file().size() : (uint64_t)c.get_output_basis_file().size(), + tm.get_elapsed_secs(), + opts.m_ktx2_mode ? c.get_ktx2_bits_per_texel() : c.get_basis_bits_per_texel()); } } else @@ -1479,6 +1791,15 @@ static bool compress_mode(command_line_params &opts) switch (ec) { + case basis_compressor::cECFailedInvalidParameters: + { + error_printf("Invalid compressor parameters (internal error)\n"); + + if (opts.m_individual) + exit_flag = false; + + break; + } case basis_compressor::cECFailedReadingSourceImages: { error_printf("Compressor failed reading a source image!\n"); @@ -1559,7 +1880,7 @@ static bool compress_mode(command_line_params &opts) fprintf(pCSV_file, "\"%s\", %u, %u, %u, %u, %u, %f, %f, %f, %f, %f, %u, %u, %f, %f, %f, %f, %f, %f, %f\n", params.m_out_filename.c_str(), - c.get_basis_file_size(), + (uint32_t)c.get_basis_file_size(), (uint32_t)c.get_stats().size(), c.get_stats()[0].m_width, c.get_stats()[0].m_height, (uint32_t)c.get_any_source_image_has_alpha(), c.get_basis_bits_per_texel(), @@ -1567,7 +1888,7 @@ static bool compress_mode(command_line_params &opts) c.get_stats()[0].m_basis_rgba_avg_psnr, c.get_stats()[0].m_basis_luma_709_psnr, c.get_stats()[0].m_best_etc1s_luma_709_psnr, - params.m_etc1s_quality_level, (int)params.m_compression_level, tm.get_elapsed_secs(), + params.m_quality_level, (int)params.m_etc1s_compression_level, tm.get_elapsed_secs(), rgb_avg_psnr_min, rgb_avg_psnr_avg, a_avg_psnr_min, a_avg_psnr_avg, luma_709_psnr_min, luma_709_psnr_avg); @@ -1591,7 +1912,7 @@ static bool compress_mode(command_line_params &opts) comp_params_vec, results); BASISU_NOTE_UNUSED(any_failed); - + for (uint32_t i = 0; i < comp_params_vec.size(); i++) { if (results[i].m_error_code != basis_compressor::cECSuccess) @@ -1600,8 +1921,8 @@ static bool compress_mode(command_line_params &opts) total_failures++; - error_printf("File %u (first source image: \"%s\", output file: \"%s\") failed with error code %i!\n", i, - comp_params_vec[i].m_source_filenames[0].c_str(), + error_printf("File %u (first source image: \"%s\", output file: \"%s\") failed with error code %i!\n", i, + comp_params_vec[i].m_source_filenames[0].c_str(), comp_params_vec[i].m_out_filename.c_str(), (int)results[i].m_error_code); } @@ -1610,11 +1931,11 @@ static bool compress_mode(command_line_params &opts) total_successes++; } } - + } // if (opts.m_parallel_compression) printf("Total successes: %u failures: %u\n", total_successes, total_failures); - + all_tm.stop(); if (total_files > 1) @@ -1625,7 +1946,7 @@ static bool compress_mode(command_line_params &opts) fclose(pCSV_file); pCSV_file = nullptr; } - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return result; @@ -1641,9 +1962,8 @@ static bool unpack_and_validate_ktx2_file( uint32_t& total_unpack_warnings, uint32_t& total_pvrtc_nonpow2_warnings) { - // TODO - (void)pCSV_file; - (void)file_index; + BASISU_NOTE_UNUSED(pCSV_file); + BASISU_NOTE_UNUSED(file_index); const bool validate_flag = (opts.m_mode == cValidate); @@ -1666,7 +1986,7 @@ static bool unpack_and_validate_ktx2_file( error_printf("ktx2_transcoder::start_transcoding() failed! File either uses an unsupported feature, is invalid, was corrupted, or this is a bug.\n"); return false; } - + printf("Resolution: %ux%u\n", dec.get_width(), dec.get_height()); fmt_printf("Block size: {}x{}\n", dec.get_block_width(), dec.get_block_height()); printf("Mipmap Levels: %u\n", dec.get_levels()); @@ -1676,10 +1996,13 @@ static bool unpack_and_validate_ktx2_file( if (dec.is_hdr()) fmt_printf("LDR to HDR upconversion nit multiplier: {}\n", dec.get_ldr_hdr_upconversion_nit_multiplier()); - + const bool is_etc1s = (dec.get_basis_tex_format() == basist::basis_tex_format::cETC1S); - + bool is_hdr = false; + //bool is_xuastc_ldr = false, is_astc_ldr = false; + + std::string fmt_str_temp; const char* pFmt_str = nullptr; switch (dec.get_basis_tex_format()) @@ -1689,7 +2012,7 @@ static bool unpack_and_validate_ktx2_file( pFmt_str = "ETC1S"; break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { pFmt_str = "UASTC_LDR_4x4"; break; @@ -1706,10 +2029,56 @@ static bool unpack_and_validate_ktx2_file( pFmt_str = "ASTC_HDR_6x6"; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { is_hdr = true; - pFmt_str = "ASTC_HDR_6x6_INTERMEDIATE"; + pFmt_str = "UASTC_HDR_6x6_INTERMEDIATE"; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + //is_xuastc_ldr = true; + + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(dec.get_basis_tex_format(), block_width, block_height); + fmt_str_temp = fmt_string("XUASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + //is_astc_ldr = true; + + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(dec.get_basis_tex_format(), block_width, block_height); + fmt_str_temp = fmt_string("ASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); break; } default: @@ -1720,7 +2089,7 @@ static bool unpack_and_validate_ktx2_file( } printf("Supercompression Format: %s\n", pFmt_str); - + printf("Supercompression Scheme: "); switch (dec.get_header().m_supercompression_scheme) { @@ -1733,9 +2102,9 @@ static bool unpack_and_validate_ktx2_file( } printf("Has Alpha: %u\n", (uint32_t)dec.get_has_alpha()); - + printf("\nKTX2 header vk_format: 0x%X (decimal %u)\n", (uint32_t)dec.get_header().m_vk_format, (uint32_t)dec.get_header().m_vk_format); - + printf("\nData Format Descriptor (DFD):\n"); printf("DFD length in bytes: %zu\n", dec.get_dfd().size()); printf("DFD color model: %u\n", dec.get_dfd_color_model()); @@ -1751,8 +2120,13 @@ static bool unpack_and_validate_ktx2_file( printf("DFD chan1: %s\n", basist::ktx2_get_etc1s_df_channel_id_str(dec.get_dfd_channel_id1())); } else + { printf("DFD chan0: %s\n", basist::ktx2_get_uastc_df_channel_id_str(dec.get_dfd_channel_id0())); + } + // For proper ASTC decoding we must know which ASTC decode profile to apply (sRGB or linear). + const bool actual_ktx2_srgb_transfer_func = (dec.get_dfd_transfer_func() == basist::KTX2_KHR_DF_TRANSFER_SRGB); + printf("DFD hex values:\n"); for (uint32_t i = 0; i < dec.get_dfd().size(); i++) { @@ -1764,6 +2138,25 @@ static bool unpack_and_validate_ktx2_file( } printf("\n"); + // the sRGB transfer function to use while unpacking astc content (ideally we want this to always match what we used during astc encoding) + bool srgb_transfer_func_astc_unpacking = actual_ktx2_srgb_transfer_func; + + // the sRGB transfer function to use when writing out files (we want to indicate to the caller if the data is sRGB or linear) + bool srgb_transfer_func_astc_writing = actual_ktx2_srgb_transfer_func; + + const bool is_uastc_ldr_4x4 = (dec.get_basis_tex_format() == basist::basis_tex_format::cUASTC_LDR_4x4); + if ((is_etc1s) || (is_uastc_ldr_4x4)) + { + // The ETC1S and UASTC LDR 4x4 transcoders supply ASTC LDR 4x4 data assuming the decoder will NOT be using the sRGB read decode profile, which is likely the most common case (in geospatial rendering scenarios). + // Note XUASTC/UASTC LDR 4x4-12x12 supports both linear and sRGB decode profiles throughout the entire pipeline (encoding/transcoding/decoding to raw pixels). + srgb_transfer_func_astc_unpacking = false; + + // This matches the behavior of our original tools. It ensures astcenc uses linear by default when reading our transcoded .KTX files. + srgb_transfer_func_astc_writing = false; + + if (actual_ktx2_srgb_transfer_func) + printf("Note: ETC1S/UASTC LDR 4x4 will always be decoded by this tool using the ASTC linear decode profile, regardless of the KTX2/.basis DFD transfer function field.\n"); + } printf("Total key values: %zu\n", dec.get_key_values().size()); for (uint32_t i = 0; i < dec.get_key_values().size(); i++) @@ -1772,13 +2165,13 @@ static bool unpack_and_validate_ktx2_file( if (dec.get_key_values()[i].m_value.size() > 256) continue; - + bool is_ascii = true; for (uint32_t j = 0; j < dec.get_key_values()[i].m_value.size(); j++) { uint8_t c = dec.get_key_values()[i].m_value[j]; - if (!( - ((c >= ' ') && (c < 0x80)) || + if (!( + ((c >= ' ') && (c < 0x80)) || ((j == dec.get_key_values()[i].m_value.size() - 1) && (!c)) )) { @@ -1851,10 +2244,10 @@ static bool unpack_and_validate_ktx2_file( error_printf("Failed retrieving image level information (%u %u %u)!\n", layer_index, level_index, face_index); return false; } - + fmt_printf("--- Level Index: {}, Layer Index: {}, Face Index: {}\n", level_info.m_level_index, level_info.m_layer_index, level_info.m_face_index); - + fmt_printf("Orig width/height: {}x{}\n", level_info.m_orig_width, level_info.m_orig_height); fmt_printf("Width/height: {}x{}\n", level_info.m_width, level_info.m_height); fmt_printf("Block width/height: {}x{}\n", level_info.m_block_width, level_info.m_block_height); @@ -1878,12 +2271,14 @@ static bool unpack_and_validate_ktx2_file( int first_format = 0; int last_format = (int)basist::transcoder_texture_format::cTFTotalTextureFormats; - if (opts.m_format_only > -1) + if (opts.m_unpack_format_only > -1) { - first_format = opts.m_format_only; + first_format = opts.m_unpack_format_only; last_format = first_format + 1; } + uint32_t transcode_flags = get_transcode_flags_from_options(opts); + for (int format_iter = first_format; format_iter < last_format; format_iter++) { basist::transcoder_texture_format tex_fmt = static_cast(format_iter); @@ -1934,7 +2329,7 @@ static bool unpack_and_validate_ktx2_file( if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGB) || (transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGBA)) { - if (!is_pow2(level_info.m_width) || !is_pow2(level_info.m_height)) + if (!is_pow2(level_info.m_orig_width) || !is_pow2(level_info.m_orig_height)) { total_pvrtc_nonpow2_warnings++; @@ -1952,13 +2347,11 @@ static bool unpack_and_validate_ktx2_file( // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - - const uint32_t decode_flags = basist::cDecodeFlagsHighQuality; - + interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, decode_flags)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, transcode_flags)) { error_printf("Failed transcoding image level (%u %u %u %u)!\n", layer_index, level_index, face_index, format_iter); return false; @@ -1966,7 +2359,7 @@ static bool unpack_and_validate_ktx2_file( double total_time = tm.get_elapsed_ms(); - printf("Transcode of layer %u level %u face %u res %ux%u format %s succeeded in %3.3f ms\n", layer_index, level_index, face_index, + printf("Transcode of layer %u level %u face %u res %ux%u format %s succeeded in %3.3f ms\n", layer_index, level_index, face_index, level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_time); } @@ -1980,14 +2373,14 @@ static bool unpack_and_validate_ktx2_file( if (validate_flag) return true; - // Now write KTX/DDS files and unpack them to individual PNG's/EXR's + // Now write KTX/DDS/ASTC files and unpack them to individual PNG's/EXR's const bool is_cubemap = (dec.get_faces() > 1); const bool is_array = (total_layers > 1); const bool is_cubemap_array = is_cubemap && is_array; const bool is_mipmapped = dec.get_levels() > 1; BASISU_NOTE_UNUSED(is_cubemap_array); BASISU_NOTE_UNUSED(is_mipmapped); - + // The maximum Direct3D array size is 2048. const uint32_t MAX_DDS_TEXARRAY_SIZE = 2048; @@ -1995,7 +2388,7 @@ static bool unpack_and_validate_ktx2_file( { const basist::transcoder_texture_format transcoder_tex_fmt = static_cast(format_iter); const basisu::texture_format tex_fmt = basis_get_basisu_texture_format(transcoder_tex_fmt); - + if (basist::basis_transcoder_format_is_uncompressed(transcoder_tex_fmt)) continue; if (!basis_is_format_supported(transcoder_tex_fmt, dec.get_basis_tex_format())) @@ -2003,8 +2396,10 @@ static bool unpack_and_validate_ktx2_file( if (transcoder_tex_fmt == basist::transcoder_texture_format::cTFBC7_ALT) continue; - // TODO: Could write DDS texture arrays. + const bool is_fmt_astc = basis_is_transcoder_texture_format_astc(transcoder_tex_fmt); + // TODO: Could write DDS texture arrays. + // No KTX tool that we know of supports cubemap arrays, so write individual cubemap files for each layer. if ((!opts.m_no_ktx) && (is_cubemap)) { @@ -2015,22 +2410,24 @@ static bool unpack_and_validate_ktx2_file( for (uint32_t face_index = 0; face_index < 6; face_index++) cubemap.push_back(gpu_images[format_iter][face_index][layer_index]); + // Write KTX1 file { std::string ktx_filename(base_filename + string_format("_transcoded_cubemap_%s_layer_%u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), layer_index)); - if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, is_fmt_astc ? srgb_transfer_func_astc_writing : actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; } printf("Wrote .KTX cubemap file \"%s\"\n", ktx_filename.c_str()); } - + + // Write .DDS file if (does_dds_support_format(cubemap[0][0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_cubemap_%s_layer_%u.dds", basist::basis_get_format_name(transcoder_tex_fmt), layer_index)); - if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2052,7 +2449,7 @@ static bool unpack_and_validate_ktx2_file( std::string dds_filename(base_filename + string_format("_transcoded_array_%s.dds", basist::basis_get_format_name(transcoder_tex_fmt))); - if (!write_compressed_texture_file(dds_filename.c_str(), tex_array, is_cubemap, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), tex_array, is_cubemap, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2061,7 +2458,7 @@ static bool unpack_and_validate_ktx2_file( } } - // Now unpack each layer and face individually and write KTX/DDS/PNG/EXR files for each + // Now unpack each layer and face individually and write KTX/DDS/ASTC/PNG/EXR/OUT files for each for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) { for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) @@ -2090,7 +2487,7 @@ static bool unpack_and_validate_ktx2_file( else ktx_filename = base_filename + string_format("_transcoded_%s_layer_%04u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), layer_index); - if (!write_compressed_texture_file(ktx_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), gi, is_fmt_astc ? srgb_transfer_func_astc_writing : actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2107,7 +2504,7 @@ static bool unpack_and_validate_ktx2_file( else dds_filename = base_filename + string_format("_transcoded_%s_layer_%04u.dds", basist::basis_get_format_name(transcoder_tex_fmt), layer_index); - if (!write_compressed_texture_file(dds_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), gi, actual_ktx2_srgb_transfer_func)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2153,11 +2550,30 @@ static bool unpack_and_validate_ktx2_file( } printf("Wrote .EXR file \"%s\"\n", rgb_filename.c_str()); } + + // Save .astc + if ((!opts.m_ktx_only) && basist::basis_is_transcoder_texture_format_astc(transcoder_tex_fmt)) + { + std::string astc_filename; + if (gi.size() > 1) + astc_filename = base_filename + string_format("_unpacked_%s_level_%u_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + astc_filename = base_filename + string_format("_unpacked_%s_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + + const gpu_image& level_g = gi[level_index]; + + if (!write_astc_file(astc_filename.c_str(), level_g.get_ptr(), level_g.get_block_width(), level_g.get_block_height(), level_info.m_width, level_info.m_height)) + { + error_printf("Failed writing to .ASTC file \"%s\"\n", astc_filename.c_str()); + return false; + } + printf("Wrote .ASTC file \"%s\"\n", astc_filename.c_str()); + } } else { image u; - if (!gi[level_index].unpack(u)) + if (!gi[level_index].unpack(u, srgb_transfer_func_astc_unpacking)) { printf("Warning: Failed unpacking GPU texture data (%u %u %u %u). Unpacking as much as possible.\n", format_iter, layer_index, level_index, face_index); total_unpack_warnings++; @@ -2182,23 +2598,7 @@ static bool unpack_and_validate_ktx2_file( } printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); } - - // Save .OUT - if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFFXT1_RGB) && (opts.m_write_out)) - { - std::string out_filename; - if (gi.size() > 1) - out_filename = base_filename + string_format("_unpacked_rgb_%s_level_%u_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); - else - out_filename = base_filename + string_format("_unpacked_rgb_%s_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); - if (!write_3dfx_out_file(out_filename.c_str(), gi[level_index])) - { - error_printf("Failed writing to OUT file \"%s\"\n", out_filename.c_str()); - return false; - } - printf("Wrote .OUT file \"%s\"\n", out_filename.c_str()); - } - + // Save alpha if (basis_transcoder_format_has_alpha(transcoder_tex_fmt) && (!opts.m_ktx_only) && (write_png)) { @@ -2227,6 +2627,41 @@ static bool unpack_and_validate_ktx2_file( printf("Wrote .PNG file \"%s\"\n", rgba_filename.c_str()); } + // Save .astc + if ((!opts.m_ktx_only) && basist::basis_is_transcoder_texture_format_astc(transcoder_tex_fmt)) + { + std::string astc_filename; + if (gi.size() > 1) + astc_filename = base_filename + string_format("_unpacked_%s_level_%u_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + astc_filename = base_filename + string_format("_unpacked_%s_face_%u_layer_%04u.astc", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + + const gpu_image& level_g = gi[level_index]; + + if (!write_astc_file(astc_filename.c_str(), level_g.get_ptr(), level_g.get_block_width(), level_g.get_block_height(), level_info.m_width, level_info.m_height)) + { + error_printf("Failed writing to .ASTC file \"%s\"\n", astc_filename.c_str()); + return false; + } + printf("Wrote .ASTC file \"%s\"\n", astc_filename.c_str()); + } + + // Save .OUT + if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFFXT1_RGB) && (opts.m_write_out)) + { + std::string out_filename; + if (gi.size() > 1) + out_filename = base_filename + string_format("_unpacked_rgb_%s_level_%u_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index); + else + out_filename = base_filename + string_format("_unpacked_rgb_%s_face_%u_layer_%04u.out", basist::basis_get_format_name(transcoder_tex_fmt), face_index, layer_index); + if (!write_3dfx_out_file(out_filename.c_str(), gi[level_index])) + { + error_printf("Failed writing to OUT file \"%s\"\n", out_filename.c_str()); + return false; + } + printf("Wrote .OUT file \"%s\"\n", out_filename.c_str()); + } + } // is_hdr } // level_index @@ -2237,7 +2672,7 @@ static bool unpack_and_validate_ktx2_file( } // format_iter - if ((opts.m_format_only == -1) && (!validate_flag)) + if ((opts.m_unpack_format_only == -1) && (!validate_flag)) { if (is_hdr) { @@ -2267,7 +2702,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2275,8 +2710,8 @@ static bool unpack_and_validate_ktx2_file( double total_transcode_time = tm.get_elapsed_ms(); - fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", - level_index, layer_index, face_index, + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); if ((!validate_flag) && (!opts.m_ktx_only)) @@ -2299,7 +2734,7 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index // RGB HALF @@ -2328,7 +2763,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, half_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2360,10 +2795,10 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index - // RGB HALF + // RGB_9E5 for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) { for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) @@ -2389,7 +2824,7 @@ static bool unpack_and_validate_ktx2_file( interval_timer tm; tm.start(); - if (!dec.transcode_image_level(level_index, layer_index, face_index, rgb9e5_img.data(), total_pixels, transcoder_tex_fmt, 0)) + if (!dec.transcode_image_level(level_index, layer_index, face_index, rgb9e5_img.data(), total_pixels, transcoder_tex_fmt, transcode_flags)) { fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); return false; @@ -2420,13 +2855,232 @@ static bool unpack_and_validate_ktx2_file( } } // face_index - } // layer_index + } // layer_index } // level_index - + } else { - // TODO: Add LDR uncompressed formats + // RGBA 32bpp + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGBA32; + + basist::ktx2_image_level_info level_info; + + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; + + image img(level_info.m_orig_width, level_info.m_orig_height); + + fill_buffer_with_random_bytes(img.get_ptr(), img.get_total_pixels() * sizeof(color_rgba)); + + interval_timer tm; + tm.start(); + + if (!dec.transcode_image_level(level_index, layer_index, face_index, img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + double total_transcode_time = tm.get_elapsed_ms(); + + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); + + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgba_filename(base_filename + fmt_string("_unpacked_rgba_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgba_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgba_filename.c_str()); + return false; + } + + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); + + std::string a_filename(base_filename + fmt_string("_unpacked_a_{}_{}_{}_{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(a_filename, img, cImageSaveGrayscale, 3)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", a_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", a_filename.c_str()); + } + + } // face_index + } // layer_index + } // level_index + + // RGB565 + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGB565; + + basist::ktx2_image_level_info level_info; + + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; + + basisu::vector packed_img(total_pixels); + + fill_buffer_with_random_bytes(packed_img.get_ptr(), packed_img.size_in_bytes()); + + interval_timer tm; + tm.start(); + + if (!dec.transcode_image_level(level_index, layer_index, face_index, packed_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + double total_transcode_time = tm.get_elapsed_ms(); + + image img(level_info.m_orig_width, level_info.m_orig_height); + + for (uint32_t y = 0; y < level_info.m_orig_height; y++) + { + for (uint32_t x = 0; x < level_info.m_orig_width; x++) + { + const uint16_t p = packed_img[x + y * level_info.m_orig_width]; + uint32_t r = p >> 11, g = (p >> 5) & 63, b = p & 31; + r = (r << 3) | (r >> 2); + g = (g << 2) | (g >> 4); + b = (b << 3) | (b >> 2); + img(x, y).set(r, g, b, 255); + } + } + + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); + + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); + + } + + } // face_index + } // layer_index + } // level_index + + // RGBA4444 + for (uint32_t level_index = 0; level_index < dec.get_levels(); level_index++) + { + for (uint32_t layer_index = 0; layer_index < total_layers; layer_index++) + { + for (uint32_t face_index = 0; face_index < dec.get_faces(); face_index++) + { + const basist::transcoder_texture_format transcoder_tex_fmt = basist::transcoder_texture_format::cTFRGBA4444; + + basist::ktx2_image_level_info level_info; + + if (!dec.get_image_level_info(level_info, level_index, layer_index, face_index)) + { + fmt_error_printf("Failed retrieving image level information ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + const uint32_t total_pixels = level_info.m_orig_width * level_info.m_orig_height; + + basisu::vector packed_img(total_pixels); + + fill_buffer_with_random_bytes(packed_img.get_ptr(), packed_img.size_in_bytes()); + + interval_timer tm; + tm.start(); + + if (!dec.transcode_image_level(level_index, layer_index, face_index, packed_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags)) + { + fmt_error_printf("Failed transcoding image level ({} {} {})!\n", layer_index, level_index, face_index); + return false; + } + + double total_transcode_time = tm.get_elapsed_ms(); + + image img(level_info.m_orig_width, level_info.m_orig_height); + + for (uint32_t y = 0; y < level_info.m_orig_height; y++) + { + for (uint32_t x = 0; x < level_info.m_orig_width; x++) + { + const uint16_t p = packed_img[x + y * level_info.m_orig_width]; + uint32_t r = p >> 12, g = (p >> 8) & 15, b = (p >> 4) & 15, a = p & 15; + r = (r << 4) | r; + g = (g << 4) | g; + b = (b << 4) | b; + a = (a << 4) | a; + img(x, y).set(r, g, b, a); + } + } + + fmt_printf("Transcode of level {} layer {} face {} res {}x{} format {} succeeded in {} ms\n", + level_index, layer_index, face_index, + level_info.m_orig_width, level_info.m_orig_height, basist::basis_get_format_name(transcoder_tex_fmt), total_transcode_time); + + if ((!validate_flag) && (!opts.m_ktx_only)) + { + std::string rgba_filename(base_filename + fmt_string("_unpacked_rgba_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgba_filename, img)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgba_filename.c_str()); + return false; + } + + std::string rgb_filename(base_filename + fmt_string("_unpacked_rgb_{}_level_{}_face_{}_layer{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(rgb_filename, img, cImageSaveIgnoreAlpha)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", rgb_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", rgb_filename.c_str()); + + std::string a_filename(base_filename + fmt_string("_unpacked_a_{}_{}_{}_{04}.png", basist::basis_get_format_name(transcoder_tex_fmt), level_index, face_index, layer_index)); + if (!save_png(a_filename, img, cImageSaveGrayscale, 3)) + { + error_printf("Failed writing to .PNG file \"%s\"\n", a_filename.c_str()); + return false; + } + printf("Wrote .PNG file \"%s\"\n", a_filename.c_str()); + } + + } // face_index + } // layer_index + } // level_index } } @@ -2437,7 +3091,7 @@ static bool unpack_and_validate_basis_file( uint32_t file_index, const std::string &base_filename, uint8_vec &basis_file_data, - command_line_params& opts, + command_line_params& opts, FILE *pCSV_file, basis_data* pGlobal_codebook_data, uint32_t &total_unpack_warnings, @@ -2460,7 +3114,7 @@ static bool unpack_and_validate_basis_file( if (!dec.validate_file_checksums(&basis_file_data[0], (uint32_t)basis_file_data.size(), true)) { error_printf("File version is unsupported, or file failed one or more CRC checks!\n"); - + return false; } } @@ -2490,6 +3144,8 @@ static bool unpack_and_validate_basis_file( bool is_hdr = false; + std::string fmt_str_temp; + const char* pFmt_str = nullptr; switch (fileinfo.m_tex_format) { @@ -2498,7 +3154,7 @@ static bool unpack_and_validate_basis_file( pFmt_str = "ETC1S"; break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { pFmt_str = "UASTC_LDR_4x4"; break; @@ -2515,10 +3171,52 @@ static bool unpack_and_validate_basis_file( pFmt_str = "ASTC_HDR_6x6"; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { is_hdr = true; - pFmt_str = "ASTC_HDR_6x6_INTERMEDIATE"; + pFmt_str = "UASTC_HDR_6x6_INTERMEDIATE"; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(fileinfo.m_tex_format, block_width, block_height); + fmt_str_temp = fmt_string("XUASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(fileinfo.m_tex_format, block_width, block_height); + fmt_str_temp = fmt_string("ASTC_LDR_{}x{}", block_width, block_height); + pFmt_str = fmt_str_temp.c_str(); break; } default: @@ -2529,18 +3227,39 @@ static bool unpack_and_validate_basis_file( } fmt_printf(" Texture format: {}\n", pFmt_str); - + printf(" Texture type: %s\n", basist::basis_get_texture_type_name(fileinfo.m_tex_type)); printf(" us per frame: %u (%f fps)\n", fileinfo.m_us_per_frame, fileinfo.m_us_per_frame ? (1.0f / ((float)fileinfo.m_us_per_frame / 1000000.0f)) : 0.0f); printf(" Total slices: %u\n", (uint32_t)fileinfo.m_slice_info.size()); printf(" Total images: %i\n", fileinfo.m_total_images); - printf(" Y Flipped: %u, Has alpha slices: %u\n", fileinfo.m_y_flipped, fileinfo.m_has_alpha_slices); + printf(" Y Flipped: %u, Has alpha slices: %u, sRGB: %u\n", fileinfo.m_y_flipped, fileinfo.m_has_alpha_slices, fileinfo.m_srgb); printf(" userdata0: 0x%X userdata1: 0x%X\n", fileinfo.m_userdata0, fileinfo.m_userdata1); printf(" Per-image mipmap levels: "); for (uint32_t i = 0; i < fileinfo.m_total_images; i++) printf("%u ", fileinfo.m_image_mipmap_levels[i]); printf("\n"); + // the sRGB transfer function to use while astc unpacking (we want this to ideally match what we used during astc encoding) + bool srgb_transfer_func_astc_unpacking = fileinfo.m_srgb; + + // the sRGB transfer function to use when writing out files (we want to indicate to the caller if the data is sRGB or linear) + bool srgb_transfer_func_astc_writing = fileinfo.m_srgb; + + const bool is_etc1s = (fileinfo.m_tex_format == basist::basis_tex_format::cETC1S); + const bool is_uastc_ldr_4x4 = (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC_LDR_4x4); + if ((is_etc1s) || (is_uastc_ldr_4x4)) + { + // The ETC1S and UASTC LDR 4x4 transcoders supply ASTC LDR 4x4 data assuming the decoder will NOT be using the sRGB read decode profile, which is likely the most common case (in geospatial rendering scenarios). + // Note XUASTC/UASTC LDR 4x4-12x12 supports both linear and sRGB decode profiles throughout the entire pipeline (encoding/transcoding/decoding to raw pixels). + srgb_transfer_func_astc_unpacking = false; + + // This matches the behavior of our original tools. It ensures astcenc uses linear by default when reading our transcoded .KTX files. + srgb_transfer_func_astc_writing = false; + + if (fileinfo.m_srgb) + printf("Note: ETC1S/UASTC LDR 4x4 will always be decoded by this tool using the ASTC linear decode profile, regardless of the KTX2/.basis DFD transfer function field.\n"); + } + uint32_t total_texels = 0; printf("\nImage info:\n"); @@ -2623,16 +3342,16 @@ static bool unpack_and_validate_basis_file( printf("start_transcoding time: %3.3f ms\n", start_transcoding_time_ms); basisu::vector< gpu_image_vec > gpu_images[(int)basist::transcoder_texture_format::cTFTotalTextureFormats]; - + double total_format_transcoding_time_ms[(int)basist::transcoder_texture_format::cTFTotalTextureFormats]; clear_obj(total_format_transcoding_time_ms); int first_format = 0; int last_format = (int)basist::transcoder_texture_format::cTFTotalTextureFormats; - if (opts.m_format_only > -1) + if (opts.m_unpack_format_only > -1) { - first_format = opts.m_format_only; + first_format = opts.m_unpack_format_only; last_format = first_format + 1; } @@ -2675,6 +3394,8 @@ static bool unpack_and_validate_basis_file( gpu_images[(int)tex_fmt][image_index].resize(fileinfo.m_image_mipmap_levels[image_index]); } + uint32_t transcode_flags = get_transcode_flags_from_options(opts); + // Now transcode the file to all supported texture formats and save mipmapped KTX files for (int format_iter = first_format; format_iter < last_format; format_iter++) { @@ -2701,30 +3422,30 @@ static bool unpack_and_validate_basis_file( if ((transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGB) || (transcoder_tex_fmt == basist::transcoder_texture_format::cTFPVRTC1_4_RGBA)) { - if (!is_pow2(level_info.m_width) || !is_pow2(level_info.m_height)) + if (!is_pow2(level_info.m_orig_width) || !is_pow2(level_info.m_orig_height)) { total_pvrtc_nonpow2_warnings++; printf("Warning: Will not transcode image %u level %u res %ux%u to PVRTC1 (one or more dimension is not a power of 2)\n", image_index, level_index, level_info.m_width, level_info.m_height); - // Can't transcode this image level to PVRTC because it's not a pow2 (we're going to support transcoding non-pow2 to the next larger pow2 soon) + // Can't transcode this image level to PVRTC because it's not a pow2 (we're going to support transcoding non-pow2 to the next "larger" pow2 soon) continue; } } basisu::texture_format tex_fmt = basis_get_basisu_texture_format(transcoder_tex_fmt); + fmt_printf("Transcoding format: {}\n", (uint32_t)tex_fmt); + gpu_image& gi = gpu_images[(int)transcoder_tex_fmt][image_index][level_index]; gi.init(tex_fmt, level_info.m_orig_width, level_info.m_orig_height); // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - - uint32_t decode_flags = basist::cDecodeFlagsHighQuality; - + tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, decode_flags)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, gi.get_ptr(), gi.get_total_blocks(), transcoder_tex_fmt, transcode_flags)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, format_iter); return false; @@ -2742,9 +3463,9 @@ static bool unpack_and_validate_basis_file( } // image_info - // Upack UASTC files seperately, to validate we can transcode slices to UASTC and unpack them to pixels. - // This is a special path because UASTC is not yet a valid transcoder_texture_format, but a lower-level block_format. - if (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC4x4) + // Upack UASTC LDR 4x4 files seperately, to validate we can transcode slices to UASTC LDR 4x4 and unpack them to pixels. + // This is a special path because UASTC LDR 4x4 is not yet a valid transcoder_texture_format, but a lower-level block_format. + if (fileinfo.m_tex_format == basist::basis_tex_format::cUASTC_LDR_4x4) { for (uint32_t image_index = 0; image_index < fileinfo.m_total_images; image_index++) { @@ -2763,12 +3484,12 @@ static bool unpack_and_validate_basis_file( // Fill the buffer with psuedo-random bytes, to help more visibly detect cases where the transcoder fails to write to part of the output. fill_buffer_with_random_bytes(gi.get_ptr(), gi.get_size_in_bytes()); - + tm.start(); if (!dec.transcode_slice( - &basis_file_data[0], (uint32_t)basis_file_data.size(), - level_info.m_first_slice_index, gi.get_ptr(), gi.get_total_blocks(), basist::block_format::cUASTC_4x4, gi.get_bytes_per_block())) + &basis_file_data[0], (uint32_t)basis_file_data.size(), + level_info.m_first_slice_index, gi.get_ptr(), gi.get_total_blocks(), basist::block_format::cUASTC_4x4, gi.get_bytes_per_block(), transcode_flags)) { error_printf("Failed transcoding image level (%u %u) to UASTC!\n", image_index, level_index); return false; @@ -2781,9 +3502,9 @@ static bool unpack_and_validate_basis_file( if ((!validate_flag) && (!opts.m_ktx_only)) { image u; - if (!gi.unpack(u)) + if (!gi.unpack(u, srgb_transfer_func_astc_unpacking)) { - error_printf("Warning: Failed unpacking GPU texture data (%u %u) to UASTC. \n", image_index, level_index); + error_printf("Warning: Failed unpacking GPU texture data (%u %u). \n", image_index, level_index); return false; } //u.crop(level_info.m_orig_width, level_info.m_orig_height); @@ -2835,6 +3556,8 @@ static bool unpack_and_validate_basis_file( if (transcoder_tex_fmt == basist::transcoder_texture_format::cTFBC7_ALT) continue; + const bool is_fmt_astc = basis_is_transcoder_texture_format_astc(transcoder_tex_fmt); + if ((!opts.m_no_ktx) && (fileinfo.m_tex_type == basist::cBASISTexTypeCubemapArray)) { // No KTX tool that we know of supports cubemap arrays, so write individual cubemap files. @@ -2844,9 +3567,10 @@ static bool unpack_and_validate_basis_file( for (uint32_t i = 0; i < 6; i++) cubemap.push_back(gpu_images[format_iter][image_index + i]); + // KTX1 { std::string ktx_filename(base_filename + string_format("_transcoded_cubemap_%s_%u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), image_index / 6)); - if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), cubemap, true, is_fmt_astc ? srgb_transfer_func_astc_writing : fileinfo.m_srgb)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2854,10 +3578,11 @@ static bool unpack_and_validate_basis_file( printf("Wrote .KTX file \"%s\"\n", ktx_filename.c_str()); } + // DDS if (does_dds_support_format(cubemap[0][0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_cubemap_%s_%u.dds", basist::basis_get_format_name(transcoder_tex_fmt), image_index / 6)); - if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), cubemap, true, fileinfo.m_srgb)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2884,9 +3609,10 @@ static bool unpack_and_validate_basis_file( if ((!opts.m_no_ktx) && (fileinfo.m_tex_type != basist::cBASISTexTypeCubemapArray)) { + // KTX1 { std::string ktx_filename(base_filename + string_format("_transcoded_%s_%04u.ktx", basist::basis_get_format_name(transcoder_tex_fmt), image_index)); - if (!write_compressed_texture_file(ktx_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(ktx_filename.c_str(), gi, is_fmt_astc ? srgb_transfer_func_astc_writing : fileinfo.m_srgb)) { error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str()); return false; @@ -2894,10 +3620,11 @@ static bool unpack_and_validate_basis_file( printf("Wrote .KTX file \"%s\"\n", ktx_filename.c_str()); } + // DDS if (does_dds_support_format(gi[0].get_format())) { std::string dds_filename(base_filename + string_format("_transcoded_%s_%04u.dds", basist::basis_get_format_name(transcoder_tex_fmt), image_index)); - if (!write_compressed_texture_file(dds_filename.c_str(), gi, true)) + if (!write_compressed_texture_file(dds_filename.c_str(), gi, fileinfo.m_srgb)) { error_printf("Failed writing DDS file \"%s\"!\n", dds_filename.c_str()); return false; @@ -2946,7 +3673,7 @@ static bool unpack_and_validate_basis_file( else { image u; - if (!gi[level_index].unpack(u)) + if (!gi[level_index].unpack(u, srgb_transfer_func_astc_unpacking)) { printf("Warning: Failed unpacking GPU texture data (%u %u %u). Unpacking as much as possible.\n", format_iter, image_index, level_index); total_unpack_warnings++; @@ -3012,7 +3739,7 @@ static bool unpack_and_validate_basis_file( } printf("Wrote .PNG file \"%s\"\n", rgba_filename.c_str()); } - + } // is_hdr } // level_index @@ -3026,7 +3753,7 @@ static bool unpack_and_validate_basis_file( uint32_t max_mipmap_levels = 0; //if (!opts.m_etc1_only) - if ((opts.m_format_only == -1) && (!validate_flag)) + if ((opts.m_unpack_format_only == -1) && (!validate_flag)) { if (is_hdr) { @@ -3052,8 +3779,8 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - half_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, + half_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3110,7 +3837,7 @@ static bool unpack_and_validate_basis_file( tm.start(); if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - half_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + half_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3167,7 +3894,7 @@ static bool unpack_and_validate_basis_file( tm.start(); if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, - rgb9e5_img.get_ptr(), total_pixels, transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + rgb9e5_img.get_ptr(), total_pixels, transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3199,8 +3926,6 @@ static bool unpack_and_validate_basis_file( } // level_index } // image_index - - } else { @@ -3225,7 +3950,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &img(0, 0).r, img.get_total_pixels(), transcoder_tex_fmt, 0, img.get_pitch(), nullptr, img.get_height())) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &img(0, 0).r, img.get_total_pixels(), transcoder_tex_fmt, transcode_flags, img.get_pitch(), nullptr, img.get_height())) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3280,7 +4005,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3343,7 +4068,7 @@ static bool unpack_and_validate_basis_file( tm.start(); - if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, 0, level_info.m_orig_width, nullptr, level_info.m_orig_height)) + if (!dec.transcode_image_level(&basis_file_data[0], (uint32_t)basis_file_data.size(), image_index, level_index, &packed_img[0], (uint32_t)packed_img.size(), transcoder_tex_fmt, transcode_flags, level_info.m_orig_width, nullptr, level_info.m_orig_height)) { error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt); return false; @@ -3391,10 +4116,10 @@ static bool unpack_and_validate_basis_file( } // level_index } // image_index - + } // is_hdr - } // if ((opts.m_format_only == -1) && (!validate_flag)) + } // if ((opts.m_unpack_format_only == -1) && (!validate_flag)) if (pCSV_file) { @@ -3432,7 +4157,7 @@ static bool unpack_and_validate_mode(command_line_params &opts) tm.start(); //const bool validate_flag = (opts.m_mode == cValidate); - + basis_data* pGlobal_codebook_data = nullptr; if (opts.m_etc1s_use_global_codebooks_file.size()) { @@ -3499,14 +4224,14 @@ static bool unpack_and_validate_mode(command_line_params &opts) delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return false; } - + bool is_ktx2 = false; if (file_data.size() >= sizeof(basist::g_ktx2_file_identifier)) { is_ktx2 = (memcmp(file_data.data(), basist::g_ktx2_file_identifier, sizeof(basist::g_ktx2_file_identifier)) == 0); } - printf("Input file \"%s\", KTX2: %u\n", pInput_filename, is_ktx2); + printf("\nInput file \"%s\", KTX2: %u\n", pInput_filename, is_ktx2); bool status; if (is_ktx2) @@ -3536,10 +4261,10 @@ static bool unpack_and_validate_mode(command_line_params &opts) if (!status) { - if (pCSV_file) + if (pCSV_file) fclose(pCSV_file); - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return false; @@ -3562,7 +4287,7 @@ static bool unpack_and_validate_mode(command_line_params &opts) fclose(pCSV_file); pCSV_file = nullptr; } - delete pGlobal_codebook_data; + delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr; return true; @@ -3608,7 +4333,7 @@ static bool hdr_compare_mode(command_line_params& opts) printf("Comparison image res: %ux%u\n", a.get_width(), a.get_height()); image_metrics im; - + im.calc_half(a, b, 0, 1, true); im.print("R "); @@ -3700,7 +4425,7 @@ static bool compare_mode(command_line_params &opts) im.calc(a, b, 0, 0, true, true); im.print("Y 601 " ); - + if (opts.m_compare_ssim) { vec4F s_rgb(compute_ssim(a, b, false, false)); @@ -3743,7 +4468,7 @@ static bool compare_mode(command_line_params &opts) save_png("delta_img_rgb.png", delta_img, cImageSaveIgnoreAlpha); printf("Wrote delta_img_rgb.png\n"); - + save_png("delta_img_a.png", delta_img, cImageSaveGrayscale, 3); printf("Wrote delta_img_a.png\n"); @@ -3886,7 +4611,7 @@ static bool compare_mode(command_line_params &opts) while ((int)strlen(tics2) < x) strcat(tics2, " "); - sprintf(buf, "0"); + snprintf(buf, sizeof(buf), "0"); strcat(tics, buf); } else if (((x & 7) == 0) || (x == X_SIZE)) @@ -3898,18 +4623,18 @@ static bool compare_mode(command_line_params &opts) strcat(tics2, " "); int v = (x - (int)X_SIZE / 2); - sprintf(buf, "%i", v / 10); + snprintf(buf, sizeof(buf), "%i", v / 10); strcat(tics, buf); if (v < 0) { if (-v < 10) - sprintf(buf, "%i", v % 10); + snprintf(buf, sizeof(buf), "%i", v % 10); else - sprintf(buf, " %i", -v % 10); + snprintf(buf, sizeof(buf), " %i", -v % 10); } else - sprintf(buf, "%i", v % 10); + snprintf(buf, sizeof(buf), "%i", v % 10); strcat(tics2, buf); } else @@ -3925,7 +4650,7 @@ static bool compare_mode(command_line_params &opts) } } // display_plot - + return true; } @@ -3964,7 +4689,7 @@ static bool split_image_mode(command_line_params& opts) } printf("Wrote file %s\n", buf); } - + return true; } @@ -4009,7 +4734,7 @@ static bool combine_images_mode(command_line_params& opts) const char* pOutput_filename = "combined.png"; if (opts.m_output_filename.size()) pOutput_filename = opts.m_output_filename.c_str(); - + if (!save_png(pOutput_filename, combined_img)) { fprintf(stderr, "Failed writing file %s\n", pOutput_filename); @@ -4051,7 +4776,7 @@ static bool tonemap_image_mode(command_line_params& opts) string_combine_path(output_filename, opts.m_output_path.c_str(), output_filename.c_str()); const char* pBasename = output_filename.c_str(); - + image srgb_img(width, height); image lin_img(width, height); @@ -4064,7 +4789,7 @@ static bool tonemap_image_mode(command_line_params& opts) p[0] = clamp(p[0], 0.0f, 1.0f); p[1] = clamp(p[1], 0.0f, 1.0f); p[2] = clamp(p[2], 0.0f, 1.0f); - + { int rc = (int)std::round(linear_to_srgb(p[0]) * 255.0f); int gc = (int)std::round(linear_to_srgb(p[1]) * 255.0f); @@ -4116,7 +4841,7 @@ static bool tonemap_image_mode(command_line_params& opts) for (int e = -6; e <= 6; e++) { const float scale = powf(2.0f, (float)e); - + tonemap_image_reinhard(tonemapped_img, hdr_img, scale, opts.m_tonemap_dither_flag); std::string filename(string_format("%s_reinhard_tonemapped_scale_%f.png", pBasename, scale)); @@ -4177,16 +4902,16 @@ static bool compsize_mode(command_line_params& opts) return true; } -const struct test_file +const struct etc1s_uastc_4x4_ldr_test_file { const char* m_pFilename; uint32_t m_etc1s_size; float m_etc1s_psnr; float m_uastc_psnr; - + uint32_t m_etc1s_128_size; float m_etc1s_128_psnr; -} g_test_files[] = +} g_etc1s_uastc_4x4_ldr_test_files[] = { { "black_1x1.png", 189, 100.0f, 100.0f, 189, 100.0f }, { "kodim01.png", 30993, 27.40f, 44.14f, 58354, 30.356064f }, @@ -4217,7 +4942,6 @@ const struct test_file { "wikipedia.png", 38961, 24.10f, 30.47f, 69558, 27.630802f }, { "alpha0.png", 766, 100.0f, 56.16f, 747, 100.000000f } }; -const uint32_t TOTAL_TEST_FILES = sizeof(g_test_files) / sizeof(g_test_files[0]); static bool test_mode_ldr(command_line_params& opts) { @@ -4236,14 +4960,16 @@ static bool test_mode_ldr(command_line_params& opts) #endif const float ETC1S_FILESIZE_THRESHOLD = .045f; - for (uint32_t i = 0; i < TOTAL_TEST_FILES; i++) + for (uint32_t i = 0; i < std::size(g_etc1s_uastc_4x4_ldr_test_files); i++) { + const auto& test_file = g_etc1s_uastc_4x4_ldr_test_files[i]; + std::string filename(opts.m_test_file_dir); if (filename.size()) { filename.push_back('/'); } - filename += std::string(g_test_files[i].m_pFilename); + filename += std::string(test_file.m_pFilename); basisu::vector source_images(1); @@ -4254,7 +4980,7 @@ static bool test_mode_ldr(command_line_params& opts) return false; } - printf("Loaded file \"%s\", dimemsions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); + printf("Loaded file \"%s\", dimensions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); image_stats stats; @@ -4264,7 +4990,7 @@ static bool test_mode_ldr(command_line_params& opts) // Test ETC1S flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus; - + { printf("**** Testing ETC1S non-OpenCL level 1\n"); @@ -4279,16 +5005,16 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S level 1 Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_size) - 1.0f); if (file_size_ratio > ETC1S_FILESIZE_THRESHOLD) { - error_printf("Expected ETC1S file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_size, (uint32_t)data_size); + error_printf("Expected ETC1S file size was %u, but got %u instead!\n", test_file.m_etc1s_size, (uint32_t)data_size); total_mismatches++; } - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_psnr) > ETC1S_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_psnr) > ETC1S_PSNR_THRESHOLD) { - error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4309,16 +5035,16 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S level 128 Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_128_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_128_size) - 1.0f); if (file_size_ratio > ETC1S_FILESIZE_THRESHOLD) { - error_printf("Expected ETC1S file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_128_size, (uint32_t)data_size); + error_printf("Expected ETC1S file size was %u, but got %u instead!\n", test_file.m_etc1s_128_size, (uint32_t)data_size); total_mismatches++; } - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_128_psnr) > ETC1S_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_128_psnr) > ETC1S_PSNR_THRESHOLD) { - error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_128_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_128_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4340,36 +5066,36 @@ static bool test_mode_ldr(command_line_params& opts) printf("ETC1S+OpenCL Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - float file_size_ratio = fabs((data_size / (float)g_test_files[i].m_etc1s_size) - 1.0f); + float file_size_ratio = fabs((data_size / (float)test_file.m_etc1s_size) - 1.0f); if (file_size_ratio > .04f) { - error_printf("Expected ETC1S+OpenCL file size was %u, but got %u instead!\n", g_test_files[i].m_etc1s_size, (uint32_t)data_size); + error_printf("Expected ETC1S+OpenCL file size was %u, but got %u instead!\n", test_file.m_etc1s_size, (uint32_t)data_size); total_mismatches++; } - if (g_test_files[i].m_etc1s_psnr == 100.0f) + if (test_file.m_etc1s_psnr == 100.0f) { // TODO if (stats.m_basis_rgba_avg_psnr < 69.0f) { - error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } - else if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_etc1s_psnr) > .2f) + else if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_etc1s_psnr) > .2f) { - error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected ETC1S+OpenCL RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } // Test UASTC { - printf("**** Testing UASTC\n"); + printf("**** Testing UASTC LDR 4x4\n"); flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus; - void* pData = basis_compress(basist::basis_tex_format::cUASTC4x4, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); + void* pData = basis_compress(basist::basis_tex_format::cUASTC_LDR_4x4, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); if (!pData) { error_printf("basis_compress() failed!\n"); @@ -4379,9 +5105,9 @@ static bool test_mode_ldr(command_line_params& opts) printf("UASTC Size: %u, PSNR: %f\n", (uint32_t)data_size, stats.m_basis_rgba_avg_psnr); - if (fabs(stats.m_basis_rgba_avg_psnr - g_test_files[i].m_uastc_psnr) > UASTC_PSNR_THRESHOLD) + if (fabs(stats.m_basis_rgba_avg_psnr - test_file.m_uastc_psnr) > UASTC_PSNR_THRESHOLD) { - error_printf("Expected UASTC RGBA Avg PSNR was %f, but got %f instead!\n", g_test_files[i].m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); + error_printf("Expected UASTC RGBA Avg PSNR was %f, but got %f instead!\n", test_file.m_etc1s_psnr, stats.m_basis_rgba_avg_psnr); total_mismatches++; } } @@ -4475,7 +5201,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te fmt_printf("test_mode_hdr: Testing basis_tex_format {}, lambda {}\n", (uint32_t)tex_fmt, lambda); uint32_t total_mismatches = 0; - + #ifdef USE_TIGHTER_TEST_TOLERANCES // The PSNR's above were created with a MSVC compiled executable, x64. Hopefully this is not too low a threshold. const float PSNR_THRESHOLD = .125f; @@ -4483,7 +5209,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te // Minor differences in how floating point code is optimized can result in slightly different generated files. const float PSNR_THRESHOLD = .3f; #endif - + double highest_delta = 0.0f; // TODO: This doesn't test all 6x6 levels, but that's fine for now. @@ -4508,8 +5234,8 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te return false; } - printf("Loaded file \"%s\", dimemsions %ux%u\n", filename.c_str(), source_image.get_width(), source_image.get_height()); - + printf("Loaded file \"%s\", dimensions %ux%u\n", filename.c_str(), source_image.get_width(), source_image.get_height()); + for (uint32_t uastc_hdr_level = 0; uastc_hdr_level <= MAX_ASTC_HDR_4x4_TEST_LEVEL; uastc_hdr_level++) { image_stats stats; @@ -4523,7 +5249,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te flags_and_quality |= uastc_hdr_level; void* pData = basis_compress(tex_fmt, - source_imagesf, flags_and_quality, lambda, + source_imagesf, flags_and_quality, lambda, &data_size, &stats); if (!pData) { @@ -4534,7 +5260,7 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te double delta1, delta2; - printf("ASTC PSNR: %f (expected %f, delta %f), BC6H PSNR: %f (expected %f, delta %f)\n", + printf("ASTC PSNR: %f (expected %f, delta %f), BC6H PSNR: %f (expected %f, delta %f)\n", stats.m_basis_rgb_avg_log2_psnr, pTest_files[i].m_level_psnr_astc[uastc_hdr_level], delta1 = fabs(stats.m_basis_rgb_avg_log2_psnr - pTest_files[i].m_level_psnr_astc[uastc_hdr_level]), stats.m_basis_rgb_avg_bc6h_log2_psnr, pTest_files[i].m_level_psnr_bc6h[uastc_hdr_level], delta2 = fabs(stats.m_basis_rgb_avg_bc6h_log2_psnr - pTest_files[i].m_level_psnr_bc6h[uastc_hdr_level])); @@ -4605,10 +5331,193 @@ static bool test_mode_hdr(command_line_params& opts, basist::basis_tex_format te return result; } +const uint32_t XUASTC_LDR_TEST_FILE_NUM_RUNS = 3; + +struct xuastc_ldr_test_file +{ + const char* m_pFilename; + + struct test_run + { + float m_dct_q; + uint32_t m_comp_size; + float m_rgba_psnr; + }; + + test_run m_test_runs[XUASTC_LDR_TEST_FILE_NUM_RUNS]; +}; + +xuastc_ldr_test_file g_xuastc_ldr_test_files_6x6[] = +{ + { "black_1x1.png", { { 100.000000f, 111, 100.000000f }, { 75.000000f, 112, 100.000000f }, { 35.000000f, 112, 100.000000f } } }, + { "kodim01.png", { { 100.000000f, 141064, 37.188324f }, { 75.000000f, 115385, 32.893822f }, { 35.000000f, 80001, 30.057878f } } }, + { "kodim02.png", { { 100.000000f, 135146, 40.280567f }, { 75.000000f, 82435, 36.618645f }, { 35.000000f, 57365, 34.556519f } } }, + { "kodim03.png", { { 100.000000f, 133654, 42.754337f }, { 75.000000f, 72161, 38.706654f }, { 35.000000f, 51462, 36.026749f } } }, + { "kodim04.png", { { 100.000000f, 138877, 40.671108f }, { 75.000000f, 84194, 36.773575f }, { 35.000000f, 61363, 34.570110f } } }, + { "kodim05.png", { { 100.000000f, 146600, 35.842682f }, { 75.000000f, 124004, 33.176735f }, { 35.000000f, 94508, 30.148199f } } }, + { "kodim06.png", { { 100.000000f, 134928, 38.721409f }, { 75.000000f, 94356, 34.459309f }, { 35.000000f, 65904, 31.435408f } } }, + { "kodim07.png", { { 100.000000f, 136807, 41.048141f }, { 75.000000f, 85150, 38.172615f }, { 35.000000f, 64387, 35.527702f } } }, + { "kodim08.png", { { 100.000000f, 145326, 35.896526f }, { 75.000000f, 119654, 33.047630f }, { 35.000000f, 92376, 29.980146f } } }, + { "kodim09.png", { { 100.000000f, 135074, 42.271267f }, { 75.000000f, 66568, 38.262554f }, { 35.000000f, 47686, 35.810940f } } }, + { "kodim10.png", { { 100.000000f, 137184, 41.879585f }, { 75.000000f, 73560, 37.980556f }, { 35.000000f, 54453, 35.449261f } } }, + { "kodim11.png", { { 100.000000f, 138275, 38.718960f }, { 75.000000f, 91902, 35.112244f }, { 35.000000f, 66243, 32.391891f } } }, + { "kodim12.png", { { 100.000000f, 132918, 42.822681f }, { 75.000000f, 71330, 38.155998f }, { 35.000000f, 49345, 35.743179f } } }, + { "kodim13.png", { { 100.000000f, 141033, 33.948277f }, { 75.000000f, 123631, 30.678318f }, { 35.000000f, 88403, 27.592640f } } }, + { "kodim14.png", { { 100.000000f, 141117, 36.902863f }, { 75.000000f, 108060, 33.896935f }, { 35.000000f, 77104, 31.451799f } } }, + { "kodim15.png", { { 100.000000f, 135981, 40.416115f }, { 75.000000f, 76564, 36.855175f }, { 35.000000f, 55002, 34.548985f } } }, + { "kodim16.png", { { 100.000000f, 134349, 42.286755f }, { 75.000000f, 80713, 36.828140f }, { 35.000000f, 55894, 33.982174f } } }, + { "kodim17.png", { { 100.000000f, 138778, 40.653671f }, { 75.000000f, 81391, 37.024017f }, { 35.000000f, 59293, 34.429058f } } }, + { "kodim18.png", { { 100.000000f, 142690, 36.400116f }, { 75.000000f, 104323, 33.398468f }, { 35.000000f, 74051, 30.714231f } } }, + { "kodim19.png", { { 100.000000f, 138584, 39.704021f }, { 75.000000f, 87574, 35.544052f }, { 35.000000f, 63776, 33.032051f } } }, + { "kodim20.png", { { 100.000000f, 121663, 41.099850f }, { 75.000000f, 64552, 37.174721f }, { 35.000000f, 44838, 34.739983f } } }, + { "kodim21.png", { { 100.000000f, 138337, 38.284393f }, { 75.000000f, 85878, 34.727512f }, { 35.000000f, 60879, 32.004494f } } }, + { "kodim22.png", { { 100.000000f, 142142, 38.583397f }, { 75.000000f, 93914, 35.047283f }, { 35.000000f, 65592, 32.702984f } } }, + { "kodim23.png", { { 100.000000f, 140280, 42.489117f }, { 75.000000f, 74579, 39.385365f }, { 35.000000f, 57354, 37.228970f } } }, + { "kodim24.png", { { 100.000000f, 138443, 36.158039f }, { 75.000000f, 101415, 33.512146f }, { 35.000000f, 75311, 30.575174f } } }, + { "white_1x1.png", { { 100.000000f, 111, 100.000000f }, { 75.000000f, 112, 100.000000f }, { 35.000000f, 112, 100.000000f } } }, + { "wikipedia.png", { { 100.000000f, 189589, 32.205330f }, { 75.000000f, 168732, 31.926851f }, { 35.000000f, 160971, 30.209082f } } }, + //{ "alpha0.png", { { 100.000000f, 1389, 49.883366f }, { 75.000000f, 1385, 49.125038f }, { 35.000000f, 1479, 42.865246f } } } // alpha0.png is minor nightmare for testing XUASTC LDR because it's very sensitive to tiny FP differences +}; + +static bool test_mode_xuastc_ldr(command_line_params& opts) +{ + uint32_t total_mismatches = 0; + + // Minor differences in how floating point code is optimized can result in slightly different generated files. + + // XUASTC LDR's IDCT is currently float - at low q's and high (>48) dB's tiny differences during decompression are noticeable + const float XUASTC_PSNR_THRESHOLD = 1.0f; + const float XUASTC_FILESIZE_THRESHOLD = .045f; + + struct run_stats + { + size_t m_comp_size; + image_stats m_stats; + }; + + basisu::vector2D< run_stats > run_image_stats((uint32_t)std::size(g_xuastc_ldr_test_files_6x6), XUASTC_LDR_TEST_FILE_NUM_RUNS); + + for (uint32_t i = 0; i < std::size(g_xuastc_ldr_test_files_6x6); i++) + { + const auto& test_file = g_xuastc_ldr_test_files_6x6[i]; + + std::string filename(opts.m_test_file_dir); + if (filename.size()) + { + filename.push_back('/'); + } + filename += std::string(test_file.m_pFilename); + + basisu::vector source_images(1); + + image& source_image = source_images[0]; + if (!load_png(filename.c_str(), source_image)) + { + error_printf("Failed loading test image \"%s\"\n", filename.c_str()); + return false; + } + + printf("Loaded file \"%s\", dimensions %ux%u has alpha: %u\n", filename.c_str(), source_image.get_width(), source_image.get_height(), source_image.has_alpha()); + + image_stats stats; + + uint32_t flags_and_quality; + + // Test XUASTC LDR + flags_and_quality = (opts.m_comp_params.m_multithreading ? cFlagThreaded : 0) | cFlagPrintStats | cFlagPrintStatus | cFlagSRGB; + + for (uint32_t run_index = 0; run_index < XUASTC_LDR_TEST_FILE_NUM_RUNS; run_index++) + { + const auto& test_run = test_file.m_test_runs[run_index]; + + float uastc_rdo_quality = 0.0f; + size_t data_size = 0; + + const uint32_t effort_level = 8; + + flags_and_quality &= ~0xFF; + flags_and_quality |= effort_level; + + if (test_run.m_dct_q < 100.0f) + { + uastc_rdo_quality = test_run.m_dct_q; + } + + basist::basis_tex_format tex_fmt = basist::basis_tex_format::cXUASTC_LDR_6x6; + + fmt_printf("**** Testing XUASTC LDR, DCT q {}, effort {}\n", test_run.m_dct_q, effort_level); + + void* pData = basis_compress(tex_fmt, source_images, flags_and_quality, uastc_rdo_quality, &data_size, &stats); + if (!pData) + { + error_printf("basis_compress() failed!\n"); + return false; + } + basis_free_data(pData); + + fmt_printf("XUASTC Size: {} (expected {}), RGBA PSNR: {3.3} dB (expected {3.3} dB)\n", + (uint32_t)data_size, test_run.m_comp_size, + stats.m_basis_rgba_avg_psnr, test_run.m_rgba_psnr); + + float file_size_ratio = fabs((data_size / (float)test_run.m_comp_size) - 1.0f); + + if (file_size_ratio > XUASTC_FILESIZE_THRESHOLD) + { + fmt_error_printf("Mismatch: Expected XUASTC LDR file size was {}, but got {} instead!\n", test_run.m_comp_size, (uint32_t)data_size); + total_mismatches++; + } + + if (fabs(stats.m_basis_rgba_avg_psnr - test_run.m_rgba_psnr) > XUASTC_PSNR_THRESHOLD) + { + fmt_error_printf("Mismatch: Expected XUASTC LDR RGBA Avg PSNR was {}, but got {} instead!\n", test_run.m_rgba_psnr, stats.m_basis_rgba_avg_psnr); + total_mismatches++; + } + + run_image_stats(i, run_index).m_comp_size = data_size; + run_image_stats(i, run_index).m_stats = stats; + } + } + +#if 0 + for (uint32_t i = 0; i < std::size(g_xuastc_ldr_test_files_6x6); i++) + { + fmt_printf("{{ \"{}\", {{", g_xuastc_ldr_test_files_6x6[i].m_pFilename); + + for (uint32_t j = 0; j < XUASTC_LDR_TEST_FILE_NUM_RUNS; j++) + { + fmt_printf(" {{ {}f, {}, {}f }", + g_xuastc_ldr_test_files_6x6[i].m_test_runs[j].m_dct_q, + run_image_stats(i, j).m_comp_size, + run_image_stats(i, j).m_stats.m_basis_rgba_avg_psnr); + + if (j != (XUASTC_LDR_TEST_FILE_NUM_RUNS - 1)) + fmt_printf(", "); + } + + fmt_printf(" } },\n"); + } +#endif + + printf("Total XUASTC LDR mismatches: %u\n", total_mismatches); + + bool result = true; + if (total_mismatches) + { + error_printf("XUASTC LDR test FAILED\n"); + result = false; + } + else + { + printf("XUASTC LDR test succeeded\n"); + } + + return result; +} + static bool clbench_mode(command_line_params& opts) { BASISU_NOTE_UNUSED(opts); - + bool opencl_failed = false; bool use_cl = basis_benchmark_etc1s_opencl(&opencl_failed); if (use_cl) @@ -4639,17 +5548,513 @@ static void force_san_failure() } #endif // FORCE_SAN_FAILURE -static int main_internal(int argc, const char **argv) +static bool peek_astc_file(const char* pFilename) { - printf("Basis Universal LDR/HDR GPU Texture Compression and Transcoding System v" BASISU_TOOL_VERSION + fmt_printf("\nExamining .astc file: \"{}\"\n", pFilename); + + vector2D blocks; + uint32_t block_width, block_height, image_width, image_height; + if (!read_astc_file(pFilename, blocks, block_width, block_height, image_width, image_height)) + { + fmt_error_printf("Failed reading .astc file!\n"); + return false; + } + + const uint32_t total_block_pixels = block_width * block_height; + + fmt_printf("Block dimensions in pixels: {}x{}, {} total pixels\n", block_width, block_height, total_block_pixels); + fmt_printf("Image dimensions in pixels: {}x{}\n", image_width, image_height); + + fmt_printf("Extra cols/rows to pad image to ASTC block dimensions: {}x{}\n", + blocks.get_width() * block_width - image_width, + blocks.get_height() * block_height - image_height); + + image dec_image_srgb(image_width, image_height); + image dec_image_linear(image_width, image_height); + imagef dec_image_float(image_width, image_height); + + uint32_t cem_hist[16] = { }; + uint32_t cem_dp_hist[16] = { }; + uint32_t cem_used_bc_hist[16] = { }; + uint32_t total_dp = 0; + + uint32_t total_solid_blocks_ldr = 0; + uint32_t total_solid_blocks_hdr = 0; + uint32_t total_normal_blocks = 0; + + uint32_t part_hist[4] = { }; + uint32_t used_endpoint_levels_hist[astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + 1] = { }; + uint32_t used_weight_levels_hist[astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + 1] = { }; + + uint32_t total_unequal_cem_blocks = 0; + uint32_t total_unequal_cem_blocks_2subsets = 0; + uint32_t total_unequal_cem_blocks_3subsets = 0; + uint32_t total_unequal_cem_blocks_4subsets = 0; + + uint32_t highest_part_seed = 0; + + int min_weight_grid_width = INT_MAX, min_weight_grid_height = INT_MAX; + int max_weight_grid_width = 0, max_weight_grid_height = 0; + + uint32_t total_ldr_blocks = 0, total_hdr_blocks = 0; + + basisu::hash_map weight_grid_histogram; + + struct log_astc_block_config_cmp_t + { + bool operator()(const astc_helpers::log_astc_block& a, + const astc_helpers::log_astc_block& b) const + { + // This only compares the ASTC configuration for equality, NOT the contents. + if (a.m_error_flag != b.m_error_flag) + return false; + if (a.m_error_flag) + return true; + + if (a.m_grid_width != b.m_grid_width) + return false; + if (a.m_grid_height != b.m_grid_height) + return false; + + if (a.m_solid_color_flag_ldr != b.m_solid_color_flag_ldr) + return false; + if (a.m_solid_color_flag_hdr != b.m_solid_color_flag_hdr) + return false; + + if (a.m_solid_color_flag_ldr || a.m_solid_color_flag_hdr) + return true; + + if (a.m_dual_plane != b.m_dual_plane) + return false; + if (a.m_color_component_selector != b.m_color_component_selector) + return false; + + if (a.m_num_partitions != b.m_num_partitions) + return false; + + if (a.m_endpoint_ise_range != b.m_endpoint_ise_range) + return false; + if (a.m_weight_ise_range != b.m_weight_ise_range) + return false; + + for (uint32_t i = 0; i < a.m_num_partitions; i++) + if (a.m_color_endpoint_modes[i] != b.m_color_endpoint_modes[i]) + return false; + + return true; + } + }; + + basisu::hash_map, log_astc_block_config_cmp_t > unique_config_histogram; + + for (uint32_t by = 0; by < blocks.get_height(); by++) + { + for (uint32_t bx = 0; bx < blocks.get_width(); bx++) + { + astc_helpers::log_astc_block log_blk; + + if (!astc_helpers::unpack_block(&blocks(bx, by), log_blk, block_width, block_height)) + { + fmt_error_printf("astc_helpers::unpack_block() failed on block {}x{}\n", bx, by); + return false; + } + + if (log_blk.m_error_flag) + { + fmt_error_printf("astc_helpers::unpack_block() returned an error flag on block {}x{}\n", bx, by); + return false; + } + + { + astc_helpers::log_astc_block scrubbed_log_blk; + memset(&scrubbed_log_blk, 0, sizeof(scrubbed_log_blk)); + + // just record the config, not the contents, so only the config hashes + scrubbed_log_blk.m_solid_color_flag_ldr = log_blk.m_solid_color_flag_ldr; + scrubbed_log_blk.m_solid_color_flag_hdr = log_blk.m_solid_color_flag_hdr; + scrubbed_log_blk.m_dual_plane = log_blk.m_dual_plane; + scrubbed_log_blk.m_color_component_selector = log_blk.m_color_component_selector; + scrubbed_log_blk.m_grid_width = log_blk.m_grid_width; + scrubbed_log_blk.m_grid_height = log_blk.m_grid_height; + scrubbed_log_blk.m_num_partitions = log_blk.m_num_partitions; + scrubbed_log_blk.m_color_endpoint_modes[0] = log_blk.m_color_endpoint_modes[0]; + scrubbed_log_blk.m_color_endpoint_modes[1] = log_blk.m_color_endpoint_modes[1]; + scrubbed_log_blk.m_color_endpoint_modes[2] = log_blk.m_color_endpoint_modes[2]; + scrubbed_log_blk.m_color_endpoint_modes[3] = log_blk.m_color_endpoint_modes[3]; + scrubbed_log_blk.m_weight_ise_range = log_blk.m_weight_ise_range; + scrubbed_log_blk.m_endpoint_ise_range = log_blk.m_endpoint_ise_range; + + auto ins_res(unique_config_histogram.insert(scrubbed_log_blk, 0)); + (ins_res.first)->second = (ins_res.first)->second + 1; + } + + bool is_hdr = log_blk.m_solid_color_flag_hdr; + + if (log_blk.m_solid_color_flag_ldr) + { + total_solid_blocks_ldr++; + total_ldr_blocks++; + } + else if (log_blk.m_solid_color_flag_hdr) + { + total_solid_blocks_hdr++; + total_hdr_blocks++; + } + else + { + total_normal_blocks++; + + min_weight_grid_width = minimum(min_weight_grid_width, log_blk.m_grid_width); + min_weight_grid_height = minimum(min_weight_grid_height, log_blk.m_grid_height); + + max_weight_grid_width = maximum(max_weight_grid_width, log_blk.m_grid_width); + max_weight_grid_height = maximum(max_weight_grid_height, log_blk.m_grid_height); + + { + uint32_t weight_grid_hash_key = log_blk.m_grid_width | (log_blk.m_grid_height << 8); + auto ins_res(weight_grid_histogram.insert(weight_grid_hash_key, 0)); + (ins_res.first)->second = (ins_res.first)->second + 1; + } + + if (log_blk.m_dual_plane) + total_dp++; + + part_hist[log_blk.m_num_partitions - 1]++; + + // For debugging seed packing bugs + highest_part_seed = basisu::maximum(highest_part_seed, log_blk.m_partition_id); + + uint32_t cur_endpoint_ofs = 0; + bool has_unequal_cems = false; + + for (uint32_t p = 0; p < log_blk.m_num_partitions; p++) + { + if (astc_helpers::is_cem_hdr(log_blk.m_color_endpoint_modes[p])) + is_hdr = true; + + cem_hist[log_blk.m_color_endpoint_modes[p]]++; + + if (log_blk.m_dual_plane) + cem_dp_hist[log_blk.m_color_endpoint_modes[p]]++; + + if ((p) && (log_blk.m_color_endpoint_modes[p] != log_blk.m_color_endpoint_modes[0])) + { + has_unequal_cems = true; + } + + if (astc_helpers::is_cem_ldr(log_blk.m_color_endpoint_modes[p])) + { + bool uses_bc = astc_helpers::used_blue_contraction(log_blk.m_color_endpoint_modes[p], log_blk.m_endpoints + cur_endpoint_ofs, log_blk.m_endpoint_ise_range); + + cem_used_bc_hist[log_blk.m_color_endpoint_modes[p]] += uses_bc; + } + + cur_endpoint_ofs += astc_helpers::get_num_cem_values(log_blk.m_color_endpoint_modes[p]); + } + + if (log_blk.m_num_partitions >= 2) + { + total_unequal_cem_blocks += has_unequal_cems; + + if (log_blk.m_num_partitions == 2) + total_unequal_cem_blocks_2subsets += has_unequal_cems; + else if (log_blk.m_num_partitions == 3) + total_unequal_cem_blocks_3subsets += has_unequal_cems; + else if (log_blk.m_num_partitions == 4) + total_unequal_cem_blocks_4subsets += has_unequal_cems; + } + + used_weight_levels_hist[open_range_check(log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE, std::size(used_weight_levels_hist))]++; + used_endpoint_levels_hist[open_range_check(log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE, std::size(used_endpoint_levels_hist))]++; + } + + if (is_hdr) + { + total_hdr_blocks++; + } + else + { + total_ldr_blocks++; + + color_rgba block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + // sRGB8 decode profile unpack + bool status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, astc_helpers::cDecodeModeSRGB8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + dec_image_srgb.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + + // linear8 decode profile unpack + status = astc_helpers::decode_block(log_blk, block_pixels, block_width, block_height, astc_helpers::cDecodeModeLDR8); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + dec_image_linear.set_block_clipped(block_pixels, bx * block_width, by * block_height, block_width, block_height); + } + + // half float unpack + { + basist::half_float block_pixels_half[astc_helpers::MAX_BLOCK_PIXELS][4]; + + bool status = astc_helpers::decode_block(log_blk, block_pixels_half, block_width, block_height, astc_helpers::cDecodeModeHDR16); + if (!status) + { + fmt_error_printf("astc_helpers::decode_block() failed on block {}x{}\n", bx, by); + return false; + } + + vec4F block_pixels_float[astc_helpers::MAX_BLOCK_PIXELS]; + for (uint32_t i = 0; i < total_block_pixels; i++) + for (uint32_t j = 0; j < 4; j++) + block_pixels_float[i][j] = basist::half_to_float(block_pixels_half[i][j]); + + dec_image_float.set_block_clipped(block_pixels_float, bx * block_width, by * block_height, block_width, block_height); + } + + } // bx + + } //by + + fmt_printf("Total LDR blocks: {}, total HDR blocks: {}\n", total_ldr_blocks, total_hdr_blocks); + + save_png("astc_decoded_srgb8_ldr.png", dec_image_srgb); + fmt_printf("Wrote astc_decoded_srgb8_ldr.png\n"); + + save_png("astc_decoded_linear8_ldr.png", dec_image_linear); + fmt_printf("Wrote astc_decoded_linear8_ldr.png\n"); + + write_exr("astc_decoded_half.exr", dec_image_float, 4, 0); + fmt_printf("Wrote astc_decoded_half.exr\n"); + + fmt_printf("\nASTC file statistics:\n"); + + const uint32_t total_blocks = (uint32_t)blocks.size(); + + fmt_printf("Total blocks: {}, total void extent LDR: {}, total void extent HDR: {}, total normal: {}\n", total_blocks, total_solid_blocks_ldr, total_solid_blocks_hdr, total_normal_blocks); + fmt_printf("Total dual plane: {} {3.2}%\n", total_dp, total_dp * 100.0f / (float)total_blocks); + + fmt_printf("Min weight grid dimensions: {}x{}\n", min_weight_grid_width, min_weight_grid_height); + fmt_printf("Max weight grid width: {}, height: {}\n", max_weight_grid_width, max_weight_grid_height); + + fmt_printf("\nPartition usage histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + fmt_printf("{}: {} {3.2}%\n", i + 1, part_hist[i], (float)part_hist[i] * 100.0f / (float)total_blocks); + + fmt_printf("\nCEM usage histogram:\n"); + for (uint32_t i = 0; i < 15; i++) + { + fmt_printf("{}: {} {3.2}%, total BC: {} {3.2}%, total DP: {} {3.2}%\n", i, + cem_hist[i], (float)cem_hist[i] * 100.0f / (float)total_blocks, + cem_used_bc_hist[i], (float)cem_used_bc_hist[i] * 100.0f / (float)total_blocks, + cem_dp_hist[i], (float)cem_dp_hist[i] * 100.0f / (float)total_blocks); + } + + fmt_printf("\nUsed endpoint ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_endpoint_levels_hist); i++) + fmt_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i), used_endpoint_levels_hist[i]); + + fmt_printf("\nUsed weight ISE levels:\n"); + for (uint32_t i = 0; i < std::size(used_weight_levels_hist); i++) + fmt_printf("{} levels: {}\n", astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i), used_weight_levels_hist[i]); + + fmt_printf("\nTotal 2+ subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks, (float)total_unequal_cem_blocks * 100.0f / (float)total_blocks); + fmt_printf("Total 2 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_2subsets, (float)total_unequal_cem_blocks_2subsets * 100.0f / (float)total_blocks); + fmt_printf("Total 3 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_3subsets, (float)total_unequal_cem_blocks_3subsets * 100.0f / (float)total_blocks); + fmt_printf("Total 4 subset blocks using unequal CEM's: {} {3.2}%\n", total_unequal_cem_blocks_4subsets, (float)total_unequal_cem_blocks_4subsets * 100.0f / (float)total_blocks); + + fmt_printf("\nHighest part ID seed: {}, 0x{0x}\n", highest_part_seed, highest_part_seed); + + fmt_printf("\nWeight grid usage histogram:\n"); + + uint64_vec v; + for (auto it = weight_grid_histogram.begin(); it != weight_grid_histogram.end(); ++it) + v.push_back(((uint64_t)it->first << 32) | it->second); + + v.sort(); + + for (uint32_t i = 0; i < v.size(); i++) + fmt_printf(" {}x{}: total blocks {}\n", (v[i] >> 32) & 0xFF, (v[i] >> 40) & 0xFF, v[i] & UINT32_MAX); + + fmt_printf("\nTotal unique ASTC configurations: {}\n", unique_config_histogram.size_u32()); + + uint32_t config_idx = 0; + for (auto it = unique_config_histogram.begin(); it != unique_config_histogram.end(); ++it) + { + const auto& l = it->first; + const uint32_t total = it->second; + + fmt_printf(" {}. Used {} {3.2}% times: Solid LDR: {} HDR: {}, Grid: {}x{}, Dual Plane: {}, CCS: {}, NumParts: {}, CEMS: {} {} {} {}, WeightISERange: {}, EndpointISERange: {}\n", + config_idx, total, float(total) * 100.0f / total_blocks, + l.m_solid_color_flag_ldr, l.m_solid_color_flag_hdr, + l.m_grid_width, l.m_grid_height, + l.m_dual_plane, l.m_color_component_selector, + l.m_num_partitions, l.m_color_endpoint_modes[0], l.m_color_endpoint_modes[1], l.m_color_endpoint_modes[2], l.m_color_endpoint_modes[3], + l.m_weight_ise_range, l.m_endpoint_ise_range); + + config_idx++; + } + + fmt_printf("Success\n"); + + return true; +} + +bool xuastc_ldr_decoder_fuzz_test() +{ + basisu::rand rnd; + rnd.seed(1); + + const uint32_t N = 16; + + interval_timer itm; + double total_time_a = 0, total_time_b = 0; + + for (uint32_t blk_size_index = 0; blk_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES; blk_size_index++) + { + const uint32_t bw = astc_helpers::g_astc_block_sizes[blk_size_index][0]; + const uint32_t bh = astc_helpers::g_astc_block_sizes[blk_size_index][1]; + + fmt_printf("Testing block size {}x{}\n", bw, bh); + + const auto& trial_modes = basist::astc_ldr_t::g_encoder_trial_modes[blk_size_index]; + + if (!trial_modes.size()) + { + assert(0); + return false; + } + + for (uint32_t j = 0; j < trial_modes.size(); j++) + { + const auto& tm = trial_modes[j]; + + astc_helpers::log_astc_block log_blk; + log_blk.clear(); + + const bool test_solid = rnd.irand(0, 63) == 0; + + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + + log_blk.m_dual_plane = tm.m_ccs_index != -1; + if (tm.m_ccs_index != -1) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + for (uint32_t s = 0; s < tm.m_num_parts; s++) + log_blk.m_color_endpoint_modes[s] = (uint8_t)tm.m_cem; + + for (uint32_t k = 0; k < N; k++) + { + if (log_blk.m_num_partitions > 1) + log_blk.m_partition_id = (uint16_t)rnd.irand(0, 1023); + + const uint32_t num_cem_endpoint_vals = astc_helpers::get_num_cem_values(tm.m_cem); + const uint32_t total_cem_endpoint_vals = num_cem_endpoint_vals * log_blk.m_num_partitions; + + for (uint32_t i = 0; i < total_cem_endpoint_vals; i++) + log_blk.m_endpoints[i] = (uint8_t)rnd.irand(0, astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range) - 1); + + const uint32_t num_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; + for (uint32_t i = 0; i < num_weight_vals; i++) + log_blk.m_weights[i] = (uint8_t)rnd.irand(0, astc_helpers::get_ise_levels(log_blk.m_weight_ise_range) - 1); + + if (test_solid) + { + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + + uint32_t r = rnd.byte(); + uint32_t g = rnd.byte(); + uint32_t b = rnd.byte(); + uint32_t a = rnd.byte(); + + log_blk.m_solid_color[0] = (uint16_t)((r << 8) | r); + log_blk.m_solid_color[1] = (uint16_t)((g << 8) | g); + log_blk.m_solid_color[2] = (uint16_t)((b << 8) | b); + log_blk.m_solid_color[3] = (uint16_t)((a << 8) | a); + } + + const bool srgb = rnd.bit(); + + basist::color32 blk_a[astc_helpers::MAX_BLOCK_PIXELS]; + clear_obj(blk_a); + + itm.start(); + + bool status_a = astc_helpers::decode_block(log_blk, blk_a, bw, bh, srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_a) + { + error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + total_time_a += itm.get_elapsed_secs(); + + basist::color32 blk_b[astc_helpers::MAX_BLOCK_PIXELS]; + clear_obj(blk_b); + + itm.start(); + + bool status_b = astc_helpers::decode_block_xuastc_ldr(log_blk, blk_b, bw, bh, srgb ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!status_b) + { + error_printf("astc_helpers::decode_block() failed\n"); + return false; + } + + total_time_b += itm.get_elapsed_secs(); + + for (uint32_t i = 0; i < bw * bh; i++) + { + if ((blk_a[i].r != blk_b[i].r) || (blk_a[i].g != blk_b[i].g) || (blk_a[i].b != blk_b[i].b) || (blk_a[i].a != blk_b[i].a)) + { + error_printf("decode block mismatch\n"); + return false; + } + } + + } // k + + } // j + + } // blk_size_index + + printf("ASTC block decoder vs. XUASTC LDR block decoding fuzz test succeeded\n"); + fmt_printf("Total time A: {}, B: {}\n", total_time_a, total_time_b); + + return true; +} + +static int main_internal(int argc, const char** argv) +{ + printf("Basis Universal LDR/HDR GPU Texture Supercompression System v" BASISU_TOOL_VERSION + #if defined(_ARM64EC_) || defined(_ARM64_) - " (ARM64)" + " (ARM64)" #elif defined(_M_IX86) - " (x86)" + " (x86)" #elif defined(_M_X64) || defined(_M_AMD64) - " (x64)" + " (x64)" +#elif defined(__wasi__) + " (WASI" + #if BASISU_WASI_THREADS + " Threaded" + #endif + ")" #endif - "\nCopyright (C) 2019-2025 Binomial LLC, All rights reserved\n"); + + "\nCopyright (C) 2019-2026 Binomial LLC, All rights reserved\n"); #ifdef FORCE_SAN_FAILURE force_san_failure(); @@ -4657,17 +6062,26 @@ static int main_internal(int argc, const char **argv) //interval_timer tm; //tm.start(); - + // See if OpenCL support has been disabled. We don't want to parse the command line until the lib is initialized bool use_opencl = false; bool opencl_force_serialization = false; + bool astc_peek_flag = false; + bool astc_fuzz_flag = false; for (int i = 1; i < argc; i++) { if ((strcmp(argv[i], "-opencl") == 0) || (strcmp(argv[i], "-clbench") == 0)) use_opencl = true; + if (strcmp(argv[i], "-opencl_serialize") == 0) opencl_force_serialization = true; + + if ((strcmp(argv[i], "-peek_astc") == 0) || (strcmp(argv[i], "-peek") == 0)) + astc_peek_flag = true; + + if (strcmp(argv[i], "-dev_astc_fuzz") == 0) + astc_fuzz_flag = true; } #ifndef BASISU_SUPPORT_OPENCL @@ -4678,16 +6092,39 @@ static int main_internal(int argc, const char **argv) #endif basisu_encoder_init(use_opencl, opencl_force_serialization); + + if (astc_fuzz_flag) + { + bool status = xuastc_ldr_decoder_fuzz_test(); + return status ? EXIT_SUCCESS : EXIT_FAILURE; + } + + if (astc_peek_flag) + { + if (argc != 3) + { + fmt_error_printf("Requires filename argument of .astc file\n"); + return EXIT_FAILURE; + } + + bool status = peek_astc_file(argv[2]); + return status ? EXIT_SUCCESS : EXIT_FAILURE; + } //printf("Encoder and transcoder libraries initialized in %3.3f ms\n", tm.get_elapsed_ms()); - + if (argc == 1) { print_usage(); return EXIT_FAILURE; } - + command_line_params opts; + +#if defined(__wasi__) && !BASISU_WASI_THREADS + opts.m_comp_params.m_multithreading = false; +#endif + if (!opts.parse(argc, argv)) { //print_usage(); @@ -4699,7 +6136,7 @@ static int main_internal(int argc, const char **argv) #else printf("No SSE, Multithreading: %u, Zstandard support: %u, OpenCL: %u\n", (uint32_t)opts.m_comp_params.m_multithreading, basist::basisu_transcoder_supports_ktx2_zstd(), opencl_is_available()); #endif - + if (!opts.process_listing_files()) return EXIT_FAILURE; @@ -4748,6 +6185,9 @@ static int main_internal(int argc, const char **argv) case cTestLDR: status = test_mode_ldr(opts); break; + case cTestXUASTCLDR: + status = test_mode_xuastc_ldr(opts); + break; case cTestHDR_4x4: status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_4x4, std::size(g_hdr_4x4_test_files), g_hdr_4x4_test_files, 0.0f); break; @@ -4755,11 +6195,11 @@ static int main_internal(int argc, const char **argv) status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6, std::size(g_hdr_6x6_test_files), g_hdr_6x6_test_files, 0.0f); break; case cTestHDR_6x6i: - status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_test_files), g_hdr_6x6i_test_files, 0.0f); - + status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_test_files), g_hdr_6x6i_test_files, 0.0f); + if (status) { - status = test_mode_hdr(opts, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_l_test_files), g_hdr_6x6i_l_test_files, 500.0f); + status = test_mode_hdr(opts, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE, std::size(g_hdr_6x6i_l_test_files), g_hdr_6x6i_l_test_files, 500.0f); } break; @@ -4786,7 +6226,7 @@ static int main_internal(int argc, const char **argv) //----------------------------------------------------------------------------------- #if CLEAR_WIN32_CONSOLE -void clear_console() +void clear_console() { //if (!IsDebuggerPresent()) // return; @@ -4810,6 +6250,39 @@ void clear_console() //----------------------------------------------------------------------------------- +// Attempt to detect AddressSanitizer (ASan) across compilers - only used for debug +// output purposes. +#ifndef DETECT_ASAN_H +#define DETECT_ASAN_H + +// Start with ASAN disabled +#undef USING_ASAN + +#if defined(__wasi__) + #define USING_ASAN 0 +#else + // --- Clang / Apple Clang: use __has_feature --- + #if defined(__has_feature) + # if __has_feature(address_sanitizer) + # define USING_ASAN 1 + # endif + #endif + + // --- GCC: __SANITIZE_ADDRESS__ --- + #if defined(__SANITIZE_ADDRESS__) + # define USING_ASAN 1 + #endif +#endif // #if defined(__wasi__) + +// If still undefined, ensure USING_ASAN is cleanly defined to 0 +#ifndef USING_ASAN +# define USING_ASAN 0 +#endif + +#endif // DETECT_ASAN_H + +//----------------------------------------------------------------------------------- + int main(int argc, const char** argv) { #ifdef _WIN32 @@ -4818,15 +6291,19 @@ int main(int argc, const char** argv) #if CLEAR_WIN32_CONSOLE clear_console(); + fmt_printf("{}\n", argv[0]); #endif #if defined(DEBUG) || defined(_DEBUG) - printf("DEBUG build\n"); + printf("DEBUG or _DEBUG defined\n"); #endif -#ifdef __SANITIZE_ADDRESS__ +#if !defined(NDEBUG) + printf("NDEBUG is NOT defined\n"); +#endif +#if USING_ASAN printf("Address sanitizer enabled\n"); #endif - + int status = EXIT_FAILURE; #if BASISU_CATCH_EXCEPTIONS @@ -4848,3 +6325,4 @@ int main(int argc, const char** argv) return status; } + diff --git a/bin/clean.bat b/bin/clean.bat index d31658f..6952615 100644 --- a/bin/clean.bat +++ b/bin/clean.bat @@ -1,7 +1,13 @@ +@echo off del *.exr del *.png -del *.ktx -del *.ktx2 del *.dds del *.astc -del *.basis +del *.tga + +for %%F in (*.ktx) do ( + if /I "%%~xF"==".ktx" ( + echo Deleting "%%F" + del "%%F" + ) +) diff --git a/encoder/3rdparty/android_astc_decomp.cpp b/encoder/3rdparty/android_astc_decomp.cpp index f850d03..75e8d82 100644 --- a/encoder/3rdparty/android_astc_decomp.cpp +++ b/encoder/3rdparty/android_astc_decomp.cpp @@ -836,11 +836,13 @@ void decodeISETritBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& m[4] = data.getNext(numBits); deUint32 T7 = data.getNext(1); +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wimplicit-fallthrough=" #endif +#endif #endif switch (numValues) { @@ -853,10 +855,12 @@ void decodeISETritBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& default: DE_ASSERT(false); } +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic pop #endif +#endif #endif const deUint32 T = (T7 << 7) | (T56 << 5) | (T4 << 4) | (T23 << 2) | (T01 << 0); @@ -902,11 +906,13 @@ void decodeISEQuintBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& m[2] = data.getNext(numBits); deUint32 Q56 = data.getNext(2); +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wimplicit-fallthrough=" #endif +#endif #endif switch (numValues) { @@ -917,10 +923,13 @@ void decodeISEQuintBlock (ISEDecodedResult* dst, int numValues, BitAccessStream& default: DE_ASSERT(false); } + +#ifndef __clang__ #ifndef __EMSCRIPTEN__ #ifdef __GNUC__ #pragma GCC diagnostic pop #endif +#endif #endif const deUint32 Q = (Q56 << 5) | (Q34 << 3) | (Q012 << 0); diff --git a/encoder/3rdparty/tinydds.h b/encoder/3rdparty/tinydds.h index 41e1d6f..b1dda65 100644 --- a/encoder/3rdparty/tinydds.h +++ b/encoder/3rdparty/tinydds.h @@ -828,7 +828,7 @@ typedef struct TinyDDS_Context { #define TINYDDS_MAKE_RIFFCODE(a, b, c, d) (a | (b << 8) | (c << 16) | (d << 24)) -static uint32_t TinyDDS_fileIdentifier = TINYDDS_MAKE_RIFFCODE('D', 'D', 'S', ' '); +//static uint32_t TinyDDS_fileIdentifier = TINYDDS_MAKE_RIFFCODE('D', 'D', 'S', ' '); static void TinyDDS_NullErrorFunc(void *user, char const *msg) { BASISU_NOTE_UNUSED(user); BASISU_NOTE_UNUSED(msg); } diff --git a/encoder/basisu_astc_hdr_6x6_enc.cpp b/encoder/basisu_astc_hdr_6x6_enc.cpp index fd1efe5..b41d69b 100644 --- a/encoder/basisu_astc_hdr_6x6_enc.cpp +++ b/encoder/basisu_astc_hdr_6x6_enc.cpp @@ -1,4 +1,16 @@ // File: basisu_astc_hdr_6x6_enc.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "basisu_astc_hdr_6x6_enc.h" #include "basisu_enc.h" #include "basisu_astc_hdr_common.h" @@ -22,13 +34,13 @@ using namespace basist::astc_6x6_hdr; namespace astc_6x6_hdr { -static void atomic_max(std::atomic& atomic_var, uint32_t new_value) +static void atomic_max(std::atomic& atomic_var, uint32_t new_value) { uint32_t current = atomic_var.load(std::memory_order_relaxed); for ( ; ; ) { uint32_t new_max = std::max(current, new_value); - if (atomic_var.compare_exchange_weak(current, new_max, std::memory_order_relaxed, std::memory_order_relaxed)) + if (atomic_var.compare_exchange_weak(current, new_max, std::memory_order_relaxed, std::memory_order_relaxed)) break; } } @@ -164,7 +176,7 @@ static float inversePQ(float E) // Highest error is for values less than SMALLEST_PQ_VAL_IN. // // Approximation is round trip lossless for 10-12 bits at [0,10000] nits: -// for x [0,1024] (SCALE=1023) or for x [0,4095] (SCALE=4096): +// for x [0,1024] (SCALE=1023) or for x [0,4095] (SCALE=4096): // round(forwardPQTab(inversePQ(x / SCALE)) * SCALE) == x // // bfloat16 has enough precision to handle 8-bit sRGB to linear conversions: @@ -176,7 +188,7 @@ const int PQ_APPROX_EXP_RANGE = (PQ_APPROX_MAX_EXP - PQ_APPROX_MIN_EXP + 1); const float SMALLEST_PQ_VAL_IN = 0.000015258829080f; const float SMALLEST_PQ_VAL = 0.000551903737f; // forwardPQ(SMALLEST_PQ_VAL_IN) -const float LARGEST_PQ_VAL = 1.251312f; +const float LARGEST_PQ_VAL = 1.251312f; float g_pq_approx_tabs[PQ_APPROX_EXP_RANGE][128]; @@ -248,7 +260,7 @@ static inline float forwardPQTab(float v) } // 100 nits = ~.5 i -// This converts absolute linear RGB light in either REC 709 or REC2020/BT2100 color gamut to ICtCp, a coding space where Ct is scaled by 2. +// This converts absolute linear RGB light in either REC 709 or REC2020/BT2100 color gamut to ICtCp, a coding space where Ct is scaled by 2. // To convert to perceptual ITP for error/distance calculations, multiply the result Ct by .5 (or set itp_flag to true). // Assumes REC 709 input, or REC 2020/BT.2100 RGB input if rec2020_bt2100_color_gamut is true. // @@ -268,7 +280,7 @@ static inline float forwardPQTab(float v) static void linear_rgb_to_ictcp(const vec3F& rgb_in, vec3F& ictcp, bool itp_flag = false, bool rec2020_bt2100_color_gamut = false) { vec3F rgb_2100(rgb_in); - + float l, m, s; if (!rec2020_bt2100_color_gamut) { @@ -613,7 +625,7 @@ struct partition_pattern_vec operator size_t() const { - return basisu::hash_hsieh(m_parts, sizeof(m_parts)); + return basist::hash_hsieh(m_parts, sizeof(m_parts)); } }; @@ -662,7 +674,7 @@ public: m_nodes[0].m_outer_node = -1; uint_vec inner_list, outer_list; - + inner_list.reserve(n / 2); outer_list.reserve(n / 2); @@ -711,8 +723,8 @@ public: enum { MaxSupportedSize = 256 + 1 }; public: - result_queue() : - m_cur_size(0) + result_queue() : + m_cur_size(0) { } @@ -765,14 +777,14 @@ public: bool pop() { - if (m_cur_size == 0) + if (m_cur_size == 0) return false; m_elements[1] = m_elements[m_cur_size--]; down_heap(1); return true; } - + float get_highest_dist() const { if (!m_cur_size) @@ -780,7 +792,7 @@ public: return top().m_dist; } - + private: result_array_type m_elements; size_t m_cur_size; @@ -814,7 +826,7 @@ public: } } }; - + void find_nearest(uint32_t num_subsets, const partition_pattern_vec& desired_pat, result_queue& results, uint32_t max_results) { assert((num_subsets >= 2) && (num_subsets <= 3)); @@ -879,7 +891,7 @@ private: if (m_nodes[node_index].m_outer_node >= 0) { - if ( (results.get_size() < max_results) || + if ( (results.get_size() < max_results) || ((m_nodes[node_index].m_dist - best_dist_to_vantage) <= results.get_highest_dist()) ) { @@ -895,7 +907,7 @@ private: if (m_nodes[node_index].m_inner_node >= 0) { - if ( (results.get_size() < max_results) || + if ( (results.get_size() < max_results) || ((best_dist_to_vantage - m_nodes[node_index].m_dist) <= results.get_highest_dist()) ) { @@ -904,13 +916,13 @@ private: } } } - + void find_nearest_at_node_non_recursive(int init_node_index, uint32_t num_desired_pats, const partition_pattern_vec* pDesired_pats, result_queue& results, uint32_t max_results) { uint_vec node_stack; node_stack.reserve(16); node_stack.push_back(init_node_index); - + do { const uint32_t node_index = node_stack.back(); @@ -985,7 +997,7 @@ private: m_nodes.resize(m_nodes.size() + 1); const uint32_t new_node_index = m_nodes.size_u32() - 1; - + m_nodes[new_node_index].m_vantage_point = pUnique_pats[root_idx.first]; m_nodes[new_node_index].m_point_index = root_idx.first; m_nodes[new_node_index].m_dist = root_idx.second; @@ -1039,17 +1051,17 @@ private: basisu::vector< std::pair > dists; dists.reserve(n); - + float_vec float_dists; float_dists.reserve(n); - + for (uint32_t pat_indices_iter = 0; pat_indices_iter < n; pat_indices_iter++) { const uint32_t split_pat_index = pat_indices[pat_indices_iter]; assert(split_pat_index < num_unique_pats); const partition_pattern_vec& trial_vantage = pUnique_pats[split_pat_index]; - + dists.resize(0); float_dists.resize(0); @@ -1060,7 +1072,7 @@ private: if (pat_index == split_pat_index) continue; - + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); dists.emplace_back(std::pair(dist, pat_index)); @@ -1080,13 +1092,13 @@ private: split_dist = (split_dist + dists[(num_dists / 2) - 1].first) * .5f; uint32_t total_inner = 0, total_outer = 0; - + for (uint32_t j = 0; j < n; j++) { const uint32_t pat_index = pat_indices[j]; if (pat_index == split_pat_index) continue; - + float dist = trial_vantage.get_distance(pUnique_pats[pat_index]); if (dist <= split_dist) @@ -1096,7 +1108,7 @@ private: } float split_metric = (float)minimum(total_inner, total_outer) / (float)maximum(total_inner, total_outer); - + if ( (split_metric > best_split_metric) || ((split_metric == best_split_metric) && (s.m_var > best_split_var)) ) { @@ -1115,7 +1127,7 @@ struct partition { uint64_t m_p; - inline partition() : + inline partition() : m_p(0) { } @@ -1145,7 +1157,7 @@ struct partition inline operator size_t() const { - return hash_hsieh((const uint8_t *)&m_p, sizeof(m_p)); + return basist::hash_hsieh((const uint8_t *)&m_p, sizeof(m_p)); } }; @@ -1172,19 +1184,19 @@ static void init_partitions2_6x6() { uint64_t p_bits = 0; uint64_t p_bits_inv = 0; - + for (uint32_t y = 0; y < 6; y++) { for (uint32_t x = 0; x < 6; x++) { uint64_t p = astc_helpers::compute_texel_partition(i, x, y, 0, 2, false); assert(p < 2); - + p_bits |= (p << (x + y * 6)); p_bits_inv |= ((1 - p) << (x + y * 6)); } } - + if (!p_bits) continue; if (p_bits == ((1ULL << 36) - 1)) @@ -1206,13 +1218,13 @@ static void init_partitions2_6x6() BASISU_NOTE_UNUSED(res); } } - + uint32_t num_unique_partitions2 = 0; - + for (const auto& r : phash) { assert(r.second < 1024); - + const uint32_t unique_index = num_unique_partitions2; assert(unique_index < NUM_UNIQUE_PARTITIONS2); @@ -1221,7 +1233,7 @@ static void init_partitions2_6x6() pat_vec[i] = (uint8_t)((r.first >> i) & 1); g_partitions2[unique_index] = pat_vec; - + assert(g_part2_unique_index_to_seed[unique_index] == r.second); g_part2_seed_to_unique_index[r.second] = unique_index; @@ -1260,7 +1272,7 @@ static bool estimate_partition2_6x6( int* pBest_parts, uint32_t num_best_parts) { const uint32_t BLOCK_W = 6, BLOCK_H = 6, BLOCK_T = BLOCK_W * BLOCK_H; - + vec3F training_vecs[BLOCK_T], mean(0.0f); for (uint32_t i = 0; i < BLOCK_T; i++) @@ -1321,10 +1333,10 @@ static bool estimate_partition2_6x6( desired_part.m_parts[i] = proj < 0.0f; } #endif - + //interval_timer tm; //tm.start(); - + #if BRUTE_FORCE_PART_SEARCH uint32_t part_similarity[NUM_UNIQUE_PARTITIONS2]; @@ -1432,7 +1444,7 @@ static bool encode_block_2_subsets( part_pixel_index[part_index][l] = (uint8_t)(x + y * BLOCK_W); part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_endpoints[2][basist::NUM_MODE11_ENDPOINTS]; @@ -1529,7 +1541,7 @@ static bool encode_block_2_subsets( grid_w, grid_h, // dest/to dimension (grid size) desired_weights, // these are dequantized weights, NOT ISE symbols, [by][bx] downsampled_weights); // [wy][wx] - + best_log_blk.m_partition_id = (uint16_t)p_seed; memcpy(best_log_blk.m_endpoints, blk_endpoints[0], num_endpoint_vals); memcpy(best_log_blk.m_endpoints + num_endpoint_vals, blk_endpoints[1], num_endpoint_vals); @@ -1589,7 +1601,7 @@ static void init_partitions3_6x6() partition3_hash_map part3_hash; part3_hash.reserve(512); - + for (uint32_t seed_index = 0; seed_index < 1024; seed_index++) { partition_pattern_vec p3; @@ -1645,6 +1657,7 @@ static bool estimate_partition3_6x6( float brightest_inten = 0.0f, darkest_inten = BIG_FLOAT_VAL; vec3F cluster_centroids[NUM_SUBSETS]; + clear_obj(cluster_centroids); for (uint32_t i = 0; i < BLOCK_T; i++) { @@ -1692,23 +1705,23 @@ static bool estimate_partition3_6x6( if ((cluster_centroids[0] == cluster_centroids[2]) || (cluster_centroids[1] == cluster_centroids[2])) return false; - + uint32_t cluster_pixels[NUM_SUBSETS][BLOCK_T]; uint32_t num_cluster_pixels[NUM_SUBSETS]; vec3F new_cluster_means[NUM_SUBSETS]; const uint32_t NUM_ITERS = 4; - + for (uint32_t s = 0; s < NUM_ITERS; s++) { memset(num_cluster_pixels, 0, sizeof(num_cluster_pixels)); - memset(new_cluster_means, 0, sizeof(new_cluster_means)); + memset((void *)new_cluster_means, 0, sizeof(new_cluster_means)); for (uint32_t i = 0; i < BLOCK_T; i++) { - float d[NUM_SUBSETS] = { - training_vecs[i].squared_distance(cluster_centroids[0]), - training_vecs[i].squared_distance(cluster_centroids[1]), + float d[NUM_SUBSETS] = { + training_vecs[i].squared_distance(cluster_centroids[0]), + training_vecs[i].squared_distance(cluster_centroids[1]), training_vecs[i].squared_distance(cluster_centroids[2]) }; float min_d = d[0]; @@ -1735,7 +1748,7 @@ static bool estimate_partition3_6x6( cluster_centroids[j] = new_cluster_means[j] / (float)num_cluster_pixels[j]; } } // s - + partition_pattern_vec desired_part; for (uint32_t p = 0; p < NUM_SUBSETS; p++) { @@ -1770,7 +1783,7 @@ static bool estimate_partition3_6x6( } // part_index; std::sort(part_similarity, part_similarity + NUM_UNIQUE_PARTITIONS3); - + for (uint32_t i = 0; i < num_best_parts; i++) pBest_parts[i] = part_similarity[i] & 0xFFFF; #else @@ -1798,15 +1811,15 @@ static bool encode_block_3_subsets( astc_hdr_codec_base_options& coptions, bool uber_mode_flag, const int* pEst_patterns, int num_est_patterns, - uint32_t comp_level, + uint32_t comp_level, opt_mode_t mode11_opt_mode) { BASISU_NOTE_UNUSED(uber_mode_flag); const uint32_t BLOCK_W = 6, BLOCK_H = 6, NUM_SUBSETS = 3; const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem); - + res.m_valid = false; - + double best_e = BIG_FLOAT_VAL; astc_helpers::log_astc_block best_log_blk; @@ -1848,7 +1861,7 @@ static bool encode_block_3_subsets( part_pixel_index[part_index][l] = (uint8_t)(x + y * BLOCK_W); part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_endpoints[NUM_SUBSETS][basist::NUM_MODE11_ENDPOINTS]; @@ -1886,7 +1899,7 @@ static bool encode_block_3_subsets( blk_endpoints[part_iter], blk_weights[part_iter], coptions, - false, best_log_blk.m_endpoint_ise_range, uber_mode_flag, false, + false, best_log_blk.m_endpoint_ise_range, uber_mode_flag, false, FIRST_MODE11_SUBMODE_INDEX, MAX_MODE11_SUBMODE_INDEX, false, mode11_opt_mode); } @@ -1915,7 +1928,7 @@ static bool encode_block_3_subsets( for (uint32_t p = 0; p < NUM_SUBSETS; p++) memcpy(best_log_blk.m_endpoints + num_endpoint_vals * p, blk_endpoints[p], num_endpoint_vals); - + memcpy(best_log_blk.m_weights, ise_weights, BLOCK_W * BLOCK_H); } } @@ -1948,7 +1961,7 @@ static bool encode_block_3_subsets( astc_helpers::log_astc_block trial_blk(best_log_blk); trial_blk.m_partition_id = (uint16_t)g_part3_unique_index_to_seed[unique_part_index]; - + for (uint32_t p = 0; p < NUM_SUBSETS; p++) memcpy(trial_blk.m_endpoints + num_endpoint_vals * p, blk_endpoints[p], num_endpoint_vals); @@ -2018,7 +2031,7 @@ static uint32_t encode_values(bitwise_coder &coder, uint32_t total_values, const uint32_t total_tq_values = 0, tq_accum = 0, tq_mul = 1; assert((total_values) && (total_values <= MAX_VALS)); - + const uint32_t ep_bits = astc_helpers::g_ise_range_table[endpoint_range][0]; const uint32_t ep_trits = astc_helpers::g_ise_range_table[endpoint_range][1]; const uint32_t ep_quints = astc_helpers::g_ise_range_table[endpoint_range][2]; @@ -2061,7 +2074,7 @@ static uint32_t encode_values(bitwise_coder &coder, uint32_t total_values, const } uint32_t total_bits_output = 0; - + for (uint32_t i = 0; i < total_tq_values; i++) { const uint32_t num_bits = ep_trits ? 8 : 7; @@ -2132,7 +2145,7 @@ static void code_block(bitwise_coder& coder, { const int unique_partition_index = g_part2_seed_to_unique_index[log_blk.m_partition_id]; assert(unique_partition_index != -1); - + coder.put_truncated_binary(unique_partition_index, NUM_UNIQUE_PARTITIONS2); } else if (log_blk.m_num_partitions == 3) @@ -2142,7 +2155,7 @@ static void code_block(bitwise_coder& coder, coder.put_truncated_binary(unique_partition_index, NUM_UNIQUE_PARTITIONS3); } - + encode_values(coder, num_endpoint_vals * log_blk.m_num_partitions, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); } @@ -2176,7 +2189,7 @@ struct smooth_map_params // 3x3 region m_max_smooth_std_dev = 100.0f; m_smooth_max_mse_scale = 13000.0f; - + // 7x7 region m_max_med_smooth_std_dev = 9.0f; m_med_smooth_max_mse_scale = 15000.0f; @@ -2244,7 +2257,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t c = 0; c < 3; c++) { const basist::half_float h = basist::float_to_half(temp_block[y][x][c]); - + pDst_block_half3[x + y * 6][c] = h; pDst_block_q16[x + y * 6][c] = (float)half_to_qlog16(h); } @@ -2265,9 +2278,9 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t i = 0; i < pCol_lists[y].n; i++) p += temp_block[pCol_lists[y].p[i].pixel][x] * pCol_lists[y].p[i].weight; - + p.clamp(0.0f, basist::ASTC_HDR_MAX_VAL); - + for (uint32_t c = 0; c < 3; c++) { const basist::half_float h = basist::float_to_half(p[c]); @@ -2277,7 +2290,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo } pDst_block_q16[x + y * 6][3] = 0.0f; - + } // x } // y } @@ -2368,7 +2381,7 @@ static void filter_block(uint32_t grid_x, uint32_t grid_y, const vec3F* pSrc_blo for (uint32_t i = 0; i < pRow_lists[x].n; i++) p += vec3F(pSrc_block[y * 6 + pRow_lists[x].p[i].pixel]) * pRow_lists[x].p[i].weight; - + temp_block[y][x] = p; } // x } // y @@ -2404,7 +2417,7 @@ static float diff_blocks(const vec4F* pA, const vec4F* pB) float diff = 0.0f; for (uint32_t i = 0; i < BLOCK_T; i++) diff += square(pA[i][0] - pB[i][0]) + square(pA[i][1] - pB[i][1]) + square(pA[i][2] - pB[i][2]); - + return diff * (1.0f / (float)BLOCK_T); } @@ -2446,13 +2459,14 @@ static void create_smooth_maps2( const uint32_t height = orig_img.get_height(); //const uint32_t total_pixels = orig_img.get_total_pixels(); const uint32_t num_comps = 3; - + if (params.m_no_mse_scaling) { smooth_block_mse_scales.set_all(1.0f); return; } - + + // TODO: - move up before the no mse scaling check (harmless as that is only a debug aid) smooth_block_mse_scales.resize(width, height); image smooth_vis, med_smooth_vis, ultra_smooth_vis; @@ -2552,7 +2566,7 @@ static void create_smooth_maps2( float yl = clampf(max_std_dev / params.m_max_ultra_smooth_std_dev, 0.0f, 1.0f); yl = powf(yl, 2.0f); - + smooth_block_mse_scales(x, y) = lerp(params.m_ultra_smooth_max_mse_scale, smooth_block_mse_scales(x, y), yl); if (params.m_debug_images) @@ -2595,12 +2609,12 @@ static float compute_pixel_mse_itp(const vec3F& orig_pixel_itp, const vec3F& com float delta_i = orig_pixel_itp[0] - comp_pixel_itp[0]; float delta_t = orig_pixel_itp[1] - comp_pixel_itp[1]; float delta_p = orig_pixel_itp[2] - comp_pixel_itp[2]; - + float err = (delta_i * delta_i) + (delta_t * delta_t) + (delta_p * delta_p); if (delta_itp_dark_adjustment) { - // We have to process a large range of inputs, including extremely dark inputs. + // We have to process a large range of inputs, including extremely dark inputs. // Artifically amplify MSE on very dark pixels - otherwise they'll be overly compressed at higher lambdas. // This is to better handle very dark signals which could be explictly overexposed. float s = bu_math::smoothstep(0.0f, REALLY_DARK_I_THRESHOLD, orig_pixel_itp[0]); @@ -2688,7 +2702,7 @@ static float compute_pixel_delta_itp(const vec3F& a, const vec3F& b, const vec3F float err = 720.0f * sqrtf((delta_i * delta_i) + (delta_t * delta_t) + (delta_p * delta_p)); float s = bu_math::smoothstep(0.0f, REALLY_DARK_I_THRESHOLD, orig[0]); - + if (delta_itp_dark_adjustment) { // This is to better handle very dark signals which could be explictly overexposed. @@ -2702,22 +2716,22 @@ static float compute_pixel_delta_itp(const vec3F& a, const vec3F& b, const vec3F struct candidate_encoding { encoding_type m_encoding_type; - + basist::half_float m_solid_color[3]; uint32_t m_run_len; vec3F m_comp_pixels[MAX_BLOCK_H][MAX_BLOCK_W]; // [y][x] vec3F m_comp_pixels_itp[MAX_BLOCK_H][MAX_BLOCK_W]; // [y][x] - + endpoint_mode m_endpoint_mode; block_mode m_block_mode; bitwise_coder m_coder; - + // The block to code, which may not be valid ASTC. This may have to be transcoded (by requantizing the weights/endpoints) before it's valid ASTC. // Note the endpoints may be coded endpoints OR transcoded endpoints, depending on the encoding type. - astc_helpers::log_astc_block m_coded_log_blk; + astc_helpers::log_astc_block m_coded_log_blk; // The block the decoder outputs. astc_helpers::log_astc_block m_decomp_log_blk; @@ -2725,9 +2739,9 @@ struct candidate_encoding int m_reuse_delta_index; // m_t can get VERY large - double m_t, m_d; + double m_t, m_d; float m_bits; - + candidate_encoding() { clear(); @@ -2758,7 +2772,7 @@ struct candidate_encoding m_coded_log_blk = rhs.m_coded_log_blk; m_decomp_log_blk = rhs.m_decomp_log_blk; m_reuse_delta_index = rhs.m_reuse_delta_index; - + return *this; } @@ -2790,19 +2804,19 @@ struct candidate_encoding m_run_len = 0; clear_obj(m_comp_pixels); - + m_endpoint_mode = endpoint_mode::cInvalid; m_block_mode = block_mode::cInvalid; m_coder.restart(); - + m_coded_log_blk.clear(); m_decomp_log_blk.clear(); m_t = 0; m_d = 0; m_bits = 0; - + m_reuse_delta_index = 0; } }; @@ -2826,7 +2840,7 @@ bool decode_astc_block(uint32_t block_w, uint32_t block_h, astc_helpers::log_ast basist::half_to_float(decoded_pixels_half4[x + y * block_w][0]), basist::half_to_float(decoded_pixels_half4[x + y * block_w][1]), basist::half_to_float(decoded_pixels_half4[x + y * block_w][2])); - } // x + } // x } //y return true; @@ -2857,12 +2871,12 @@ static bool decode_file(const uint8_vec& comp_data, vector2D MAX_ASTC_HDR_6X6_DIM) || (height > MAX_ASTC_HDR_6X6_DIM)) return false; @@ -2879,11 +2893,11 @@ static bool decode_file(const uint8_vec& comp_data, vector2D num_blocks_remaining) return false; - + uint32_t prev_bx = cur_bx, prev_by = cur_by; if (cur_bx) @@ -2983,7 +2997,7 @@ static bool decode_file(const uint8_vec& comp_data, vector2D { const uint32_t width = src_img.get_width(); const uint32_t height = src_img.get_height(); - + if (pPacked_bc6h_img) pPacked_bc6h_img->resize(width, height); @@ -3369,7 +3383,7 @@ static bool pack_bc6h_image(const imagef &src_img, vector2D const uint32_t num_blocks_y = src_img.get_block_height(4); bc6h_blocks.resize(num_blocks_x, num_blocks_y); - + for (uint32_t by = 0; by < num_blocks_y; by++) { for (uint32_t bx = 0; bx < num_blocks_x; bx++) @@ -3415,7 +3429,7 @@ static bool pack_bc6h_image(const imagef &src_img, vector2D fmt_error_printf("unpack_bc6h() failed\n"); return false; } - + for (uint32_t y = 0; y < 4; y++) { for (uint32_t x = 0; x < 4; x++) @@ -3457,7 +3471,7 @@ static void estimate_partitions_mode7_and_11( uint32_t num_pats_to_examine, const uint32_t* pUnique_pat_indices_to_examine, // indices of pats to examine const vec3F *pHalf_pixels_as_floats, // block's half pixel values casted to floats const astc_hdr_codec_base_options& coptions, // options - uint32_t num_desired_pats, + uint32_t num_desired_pats, int *pDesired_pat_indices_mode11, int *pDesired_pat_indices_mode7) // output indices { BASISU_NOTE_UNUSED(coptions); @@ -3480,7 +3494,7 @@ static void estimate_partitions_mode7_and_11( candidate_res mode7_candidates[MAX_CANDIDATES]; const vec3F grayscale_axis(0.5773502691f); - + for (uint32_t examine_iter = 0; examine_iter < num_pats_to_examine; examine_iter++) { const uint32_t unique_part_index = pUnique_pat_indices_to_examine[examine_iter]; @@ -3506,7 +3520,7 @@ static void estimate_partitions_mode7_and_11( } // x } // y - + for (uint32_t i = 0; i < num_parts; i++) { assert(part_total_texels[i]); @@ -3542,7 +3556,7 @@ static void estimate_partitions_mode7_and_11( for (uint32_t part_index = 0; part_index < num_parts; part_index++) total_variance[part_index] = part_cov[part_index][0] + part_cov[part_index][3] + part_cov[part_index][5]; - vec3F part_axis[MAX_PARTS]; + //vec3F part_axis[MAX_PARTS]; float mode11_eigenvalue_est[MAX_PARTS]; // For each partition, compute the variance along the principle axis float mode7_eigenvalue_est[MAX_PARTS]; // For each partition, compute the variance along the principle axis @@ -3551,7 +3565,7 @@ static void estimate_partitions_mode7_and_11( float* pCov = &part_cov[part_index][0]; float xr = .9f, xg = 1.0f, xb = .7f; - + const uint32_t NUM_POWER_ITERS = 4; for (uint32_t iter = 0; iter < NUM_POWER_ITERS; iter++) { @@ -3564,7 +3578,7 @@ static void estimate_partitions_mode7_and_11( if (m >= 1e-10f) { m = 1.0f / m; - + r *= m; g *= m; b *= m; @@ -3576,7 +3590,7 @@ static void estimate_partitions_mode7_and_11( } float len_sq = xr * xr + xg * xg + xb * xb; - + if (len_sq < 1e-10f) { xr = grayscale_axis[0]; @@ -3591,7 +3605,7 @@ static void estimate_partitions_mode7_and_11( xg *= len_sq; xb *= len_sq; } - + { // Transform the principle axis by the covariance matrix, which will scale the vector by its eigenvalue (the variance of the dataset projected onto the principle axis). float r = xr * pCov[0] + xg * pCov[1] + xb * pCov[2]; @@ -3602,13 +3616,13 @@ static void estimate_partitions_mode7_and_11( // The result is the variance along the principle axis. //float z1 = sqrtf(r * r + g * g + b * b); // this works with the principle axis //float z2 = r * xr + g * xg + b * xb; // compute length projected along xr,xg,xb - + mode11_eigenvalue_est[part_index] = r * xr + g * xg + b * xb; } { const float yrgb = grayscale_axis[0]; - + // Transform the grayscale axis by the covariance matrix, which will scale the vector by the eigenvalue (which is the variance of the dataset projected onto this vector). float r = yrgb * pCov[0] + yrgb * pCov[1] + yrgb * pCov[2]; float g = yrgb * pCov[1] + yrgb * pCov[3] + yrgb * pCov[4]; @@ -3618,7 +3632,7 @@ static void estimate_partitions_mode7_and_11( } } // part_index - + // Compute the total variance (squared error) of the other 2 axes by subtracting the total variance of all channels by the variance of the principle axis. // TODO: Could also compute the ratio of the principle axis's variance vs. the total variance. float mode11_total_sq_dist_to_line_alt = 0.0f; @@ -3731,7 +3745,7 @@ static void estimate_partitions_mode7( } vec3F part_axis(0.5773502691f); - + // TODO: This total distance can be computed rapidly. First compute the total variance of each channel (sum the diag entries of the covar matrix), // then compute the principle eigenvalue, and subtract. The result is the variance of the projection distances. float total_sq_dist_to_line = 0.0f; @@ -3797,7 +3811,7 @@ static float calc_deblocking_penalty_itp( const vec3F& q_pixel_itp = pass_src_img_itp(qx, qy); const vec3F &d_pixel_itp = candidate.m_comp_pixels_itp[qy - by * 6][qx - bx * 6]; // compressed block - + vec3F orig_delta_v(o_pixel_itp - q_pixel_itp); total_orig_mse += square(orig_delta_v[0]) + square(orig_delta_v[1]) + square(orig_delta_v[2]); @@ -3861,15 +3875,15 @@ static bool calc_strip_size( else { rows_per_strip = (num_blocks_y / total_strips) & ~1; - + if (rows_per_strip < 2) rows_per_strip = 2;// num_blocks_y; } - + assert((rows_per_strip == num_blocks_y) || ((rows_per_strip & 1) == 0)); total_strips = (num_blocks_y + rows_per_strip - 1) / rows_per_strip; - + if (global_cfg.m_debug_output) { fmt_printf("num_blocks_y: {}, total_threads : {}, Total strips : {}\n", num_blocks_y, total_threads, total_strips); @@ -3979,7 +3993,7 @@ struct uastc_hdr_6x6_debug_state std::mutex m_vis_image_mutex; std::atomic m_comp_level_hist[ASTC_HDR_6X6_MAX_COMP_LEVEL + 1]; - + std::atomic m_total_jnd_replacements; std::mutex m_stats_mutex; @@ -3995,7 +4009,7 @@ struct uastc_hdr_6x6_debug_state } } } - + void init(uint32_t width, uint32_t height) { m_stat_vis.resize(width, height); @@ -4009,7 +4023,7 @@ struct uastc_hdr_6x6_debug_state basisu::clear_obj(m_endpoint_mode_hist); basisu::clear_obj(m_block_mode_hist); basisu::clear_obj(m_block_mode_total_bits); - + for (uint32_t i = 0; i < (uint32_t)block_mode::cBMTotalModes; i++) { for (uint32_t j = 0; j < 3; j++) @@ -4036,7 +4050,7 @@ struct uastc_hdr_6x6_debug_state for (uint32_t i = 0; i < std::size(m_total_part2_stats); i++) m_total_part2_stats[i].store(0); - + for (uint32_t i = 0; i < std::size(m_dp_stats); i++) m_dp_stats[i].store(0); @@ -4160,9 +4174,9 @@ struct uastc_hdr_6x6_debug_state struct uastc_hdr_6x6_encode_state { astc_hdr_codec_base_options master_coptions; - + imagef src_img; - + imagef src_img_filtered1; imagef src_img_filtered2; @@ -4188,7 +4202,7 @@ static bool compress_strip_task( { BASISU_NOTE_UNUSED(num_blocks_y); BASISU_NOTE_UNUSED(total_strips); - + vec3F prev_comp_pixels[BLOCK_H][BLOCK_W]; // [y][x] basisu::clear_obj(prev_comp_pixels); @@ -4427,15 +4441,15 @@ static bool compress_strip_task( for (uint32_t i = 0; i < 3; i++) { #if 0 - // 9/5/2025, wrong metric, we're iterating channels pairs here, not individual channels. + // 9/5/2025, wrong metric, we're iterating channels pairs here, not individual channels. // On 3 active channel blocks this causes no difference. - if (half_comp_stats[i].m_range > 0.0f) + if (half_comp_stats[i].m_range > 0.0f) #else static const uint8_t s_chan_pairs[3][2] = { {0, 1}, {0, 2}, {1, 2} }; - + const uint32_t chanA = s_chan_pairs[i][0]; const uint32_t chanB = s_chan_pairs[i][1]; - + if ((half_comp_stats[chanA].m_range > 0.0f) && (half_comp_stats[chanB].m_range > 0.0f)) #endif { @@ -4670,7 +4684,7 @@ static bool compress_strip_task( } // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } else if (prev_coded_log_blk.m_num_partitions == 2) { @@ -4697,7 +4711,7 @@ static bool compress_strip_task( part_half_pixels[part_index][l] = half_pixels[y][x]; part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_weights[2][BLOCK_W * BLOCK_H]; @@ -4748,7 +4762,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_astc_weights(num_grid_samples, coded_log_blk.m_weights, coded_log_blk.m_weight_ise_range, transcode_weights, decomp_log_blk.m_weight_ise_range); // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } else if (prev_coded_log_blk.m_num_partitions == 3) { @@ -4775,7 +4789,7 @@ static bool compress_strip_task( part_half_pixels[part_index][l] = half_pixels[y][x]; part_total_pixels[part_index] = l + 1; - } // x + } // x } // y uint8_t blk_weights[3][BLOCK_W * BLOCK_H]; @@ -4818,7 +4832,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_astc_weights(num_grid_samples, coded_log_blk.m_weights, coded_log_blk.m_weight_ise_range, transcode_weights, decomp_log_blk.m_weight_ise_range); // Create the block the decoder would transcode into. - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_log_blk, false); } if (!validate_log_blk(decomp_log_blk)) @@ -5361,7 +5375,7 @@ static bool compress_strip_task( memcpy(decomp_blk.m_endpoints, transcode_endpoints, num_endpoint_vals); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -5593,7 +5607,7 @@ static bool compress_strip_task( memcpy(decomp_blk.m_endpoints, transcode_endpoints, num_endpoint_vals); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -5915,7 +5929,7 @@ static bool compress_strip_task( basist::astc_6x6_hdr::requantize_ise_endpoints(mode_desc.m_cem, mode_desc.m_endpoint_ise_range, coded_log_blk.m_endpoints, mode_desc.m_transcode_endpoint_ise_range, decomp_blk.m_endpoints); - copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk); + copy_weight_grid(dual_plane, grid_x, grid_y, transcode_weights, decomp_blk, false); if (!validate_log_blk(decomp_blk)) { @@ -6117,13 +6131,13 @@ static bool compress_strip_task( mode_penalty *= (complex_block ? RUN_PENALTY * 2.0f : RUN_PENALTY); float candidate_bits = (float)candidate.m_coder.get_total_bits(); - + double candidate_d = (double)candidate_mse * mode_penalty; const float D_POWER = 2.0f; - + // this value can get VERY large after squaring on random (fuzzed) HDR inputs - double candidate_t = perceptual_scale * pow(candidate_d, D_POWER) + candidate_bits * (global_cfg.m_lambda * 1000.0f); + double candidate_t = perceptual_scale * pow(candidate_d, D_POWER) + candidate_bits * (global_cfg.m_lambda * 1000.0f); candidate.m_t = candidate_t; candidate.m_d = candidate_d; @@ -6140,7 +6154,7 @@ static bool compress_strip_task( if (best_candidate_index < 0) { assert(0); - + // Should never happen best_candidate_index = 0; } @@ -6160,7 +6174,7 @@ static bool compress_strip_task( debug_state.m_total_gaussian2_blocks.fetch_add(1, std::memory_order_relaxed); continue; } - + if (global_cfg.m_rdo_candidate_diversity_boost) { // candidate diversity boosting - consider candidates along/near the Pareto front @@ -6390,7 +6404,7 @@ static bool compress_strip_task( const uint32_t p = pat[x + y * 6]; debug_state.m_part_vis.set_clipped(bx * 6 + x, by * 6 + y, color_rgba(p ? 100 : 0, 128, p ? 100 : 0, 255)); } // x - } // y + } // y } else if (best_candidate.m_decomp_log_blk.m_num_partitions == 3) { @@ -6413,7 +6427,7 @@ static bool compress_strip_task( c.set(0, 100, 150, 255); debug_state.m_part_vis.set_clipped(bx * 6 + x, by * 6 + y, c); } // x - } // y + } // y } else if (best_candidate.m_decomp_log_blk.m_dual_plane) { @@ -6526,7 +6540,7 @@ void global_init() tm.start(); init_pq_tables(); - + init_partitions2_6x6(); init_partitions3_6x6(); @@ -6543,7 +6557,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa assert(g_initialized); if (!g_initialized) return false; - + assert(pJob_pool); if (orig_global_cfg.m_debug_output) @@ -6589,17 +6603,17 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa f = basist::ASTC_HDR_MAX_VAL; enc_state.src_img(x, y)[c] = f; - + } // c - + } // x } // y - + if (global_cfg.m_debug_images) { write_exr((global_cfg.m_debug_image_prefix + "orig.exr").c_str(), enc_state.src_img, 3, 0); } - + image src_img_compressed; tonemap_image_compressive2(src_img_compressed, enc_state.src_img); @@ -6624,10 +6638,10 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa enc_state.src_img_filtered1.resize(width, height); image_resample(enc_state.src_img, enc_state.src_img_filtered1, "gaussian", global_cfg.m_gaussian1_strength); //1.45f); - + enc_state.src_img_filtered2.resize(width, height); image_resample(enc_state.src_img, enc_state.src_img_filtered2, "gaussian", global_cfg.m_gaussian2_strength); //1.83f); - + if (global_cfg.m_debug_images) { write_exr((global_cfg.m_debug_image_prefix + "blurred1.exr").c_str(), enc_state.src_img_filtered1, 3, 0); @@ -6639,10 +6653,10 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa enc_state.src_img_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img, enc_state.src_img_itp, global_cfg); - + enc_state.src_img_filtered1_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img_filtered1, enc_state.src_img_filtered1_itp, global_cfg); - + enc_state.src_img_filtered2_itp.resize(width, height); convet_rgb_image_to_itp(enc_state.src_img_filtered2, enc_state.src_img_filtered2_itp, global_cfg); @@ -6655,20 +6669,20 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("compress_photo: Failed computing strip sizes\n"); return false; } - + if (global_cfg.m_debug_output) fmt_printf("lambda: {}, comp_level: {}, highest_comp_level: {}, extra patterns: {}\n", global_cfg.m_lambda, global_cfg.m_master_comp_level, global_cfg.m_highest_comp_level, global_cfg.m_extra_patterns_flag); - + enc_state.coded_blocks.resize(num_blocks_x, num_blocks_y); - + bitwise_coder coded_bits; - coded_bits.put_bits(0xABCD, 16); + coded_bits.put_bits(UASTC_6x6_HDR_SIG1, 16); coded_bits.put_bits(width, 16); coded_bits.put_bits(height, 16); - + enc_state.packed_img.resize(width, height); - + enc_state.strip_bits.resize(total_strips); enc_state.final_astc_blocks.resize(num_blocks_x, num_blocks_y); @@ -6679,7 +6693,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa debug_state.init(width, height); else debug_state.init(0, 0); - + interval_timer tm; tm.start(); @@ -6689,7 +6703,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t strip_index = 0; strip_index < total_strips; strip_index++) { const uint32_t strip_first_by = strip_index * rows_per_strip; - + uint32_t strip_last_by = minimum(strip_first_by + rows_per_strip - 1, num_blocks_y); if (strip_index == (total_strips - 1)) strip_last_by = num_blocks_y - 1; @@ -6715,7 +6729,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa if (any_failed_flag) break; - + } // strip_index pJob_pool->wait_for_all(); @@ -6725,7 +6739,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("One or more strips failed during compression\n"); return false; } - + if (global_cfg.m_debug_output) fmt_printf("Encoding time: {} secs\n", tm.get_elapsed_secs()); @@ -6744,7 +6758,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t i = 0; i < total_strips; i++) coded_bits.append(enc_state.strip_bits[i]); - + coded_bits.put_bits(0xA742, 16); coded_bits.flush(); @@ -6753,13 +6767,13 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa { write_exr((global_cfg.m_output_image_prefix + "comp.exr").c_str(), enc_state.packed_img, 3, 0); } - + if (global_cfg.m_debug_output) fmt_printf("\nTotal intermediate output bits/pixel: {3.4}\n", (float)coded_bits.get_total_bits() / (float)(width * height)); vector2D decoded_blocks1; vector2D decoded_blocks2; - + if (global_cfg.m_debug_output) fmt_printf("decode_file\n"); @@ -6856,7 +6870,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa fmt_error_printf("unpack_physical_astc_block() failed\n"); return false; } - + unpacked_astc_img.set_block_clipped(pixels, x * BLOCK_W, y * BLOCK_H, BLOCK_W, BLOCK_H); vec4F pixels_google[MAX_BLOCK_W * MAX_BLOCK_H]; @@ -6879,7 +6893,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } } } - + if (global_cfg.m_debug_output) fmt_printf("\nUnpack succeeded\n"); @@ -6887,9 +6901,9 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa { vector2D bc6h_blocks; - + fast_bc6h_params enc_params; - + bool pack_status = pack_bc6h_image(unpacked_astc_img, bc6h_blocks, &unpacked_bc6h_img, enc_params); if (!pack_status) { @@ -6898,7 +6912,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } unpacked_bc6h_img.crop(width, height); - + if (global_cfg.m_output_images) { write_exr((global_cfg.m_output_image_prefix + "unpacked_bc6h.exr").c_str(), unpacked_bc6h_img, 3, 0); @@ -6907,7 +6921,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa unpacked_astc_img.crop(width, height); unpacked_astc_google_img.crop(width, height); - + if (global_cfg.m_output_images) { write_exr((global_cfg.m_output_image_prefix + "unpacked_astc.exr").c_str(), unpacked_astc_img, 3, 0); @@ -6932,7 +6946,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa im.print_hp(); } } - + metrics.m_im_astc_log2.calc(enc_state.src_img, unpacked_astc_img, 0, 3, true, true); if (global_cfg.m_debug_output) @@ -6982,7 +6996,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa for (uint32_t i = 0; i < 3; i++) { im.calc(enc_state.src_img, unpacked_bc6h_img, i, 1, true, true); - + if (global_cfg.m_debug_output) { printf("%c: ", "RGBA"[i]); @@ -7004,14 +7018,14 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa if (global_cfg.m_image_stats) { image_metrics im; - + if (global_cfg.m_debug_output) printf("BC6H half float space error metrics (a piecewise linear approximation of log2 error):\n"); for (uint32_t i = 0; i < 3; i++) { im.calc_half(enc_state.src_img, unpacked_bc6h_img, i, 1, true); - + if (global_cfg.m_debug_output) { printf("%c: ", "RGBA"[i]); @@ -7020,7 +7034,7 @@ bool compress_photo(const basisu::imagef &orig_src_img, const astc_hdr_6x6_globa } metrics.m_im_bc6h_half.calc_half(enc_state.src_img, unpacked_bc6h_img, 0, 3, true); - + if (global_cfg.m_debug_output) { printf("RGB: "); diff --git a/encoder/basisu_astc_hdr_6x6_enc.h b/encoder/basisu_astc_hdr_6x6_enc.h index fe89c5b..bcc4bfd 100644 --- a/encoder/basisu_astc_hdr_6x6_enc.h +++ b/encoder/basisu_astc_hdr_6x6_enc.h @@ -5,12 +5,13 @@ namespace astc_6x6_hdr { + const uint32_t ASTC_HDR_6X6_DEF_USER_COMP_LEVEL = 2; const uint32_t ASTC_HDR_6X6_MAX_USER_COMP_LEVEL = 12; - + const uint32_t ASTC_HDR_6X6_MAX_COMP_LEVEL = 4; - + const float LDR_BLACK_BIAS = 0.0f;// .49f; - + // Note: This struct is copied several times, so do not place any heavyweight objects in here. struct astc_hdr_6x6_global_config { @@ -18,9 +19,11 @@ namespace astc_6x6_hdr // This encoder computes colorspace error in the ICtCp (or more accurately the delta ITP, where CT is scaled by .5 vs. ICtCp to become T) colorspace, so getting this correct is important. // By default the encoder assumes the input is in absolute luminance (in nits or candela per square meter, cd/m^2), specified as positive-only linear light RGB, using the REC 709 colorspace gamut (but NOT the sRGB transfer function, i.e. linear light). // If the m_rec2020_bt2100_color_gamut flag is true, the input colorspace is treated as REC 2020/BT.2100 (which is wider than 709). - // For SDR/LDR->HDR upconversion, the REC 709 sRGB input should be converted to linear light (sRGB->linear) and the resulting normalized linear RGB values scaled by either 80 or 100 nits (the luminance of a typical SDR monitor). + // For SDR/LDR->HDR upconversion, the REC 709 sRGB input should be converted to linear light (sRGB->linear) and the resulting normalized linear RGB values scaled by either 80 or 100 nits (the luminance of a typical SDR monitor). // SDR upconversion to normalized [0,1] (i.e. non-absolute) luminances may work but is not supported because ITP errors will not be predicted correctly. - bool m_rec2020_bt2100_color_gamut = false; + // 11/3/2025: This flag is always copied straight into the output KTX2 DFD colorspace, even for non-HDR formats. + // TODO: Move this parameter to reflect this. + bool m_rec2020_bt2100_color_gamut = false; // levels 0-3 normal levels, 4=exhaustive uint32_t m_master_comp_level = 0; @@ -35,13 +38,13 @@ namespace astc_6x6_hdr float m_jnd_delta_itp_thresh = .75f; bool m_force_one_strip = false; - + bool m_gaussian1_fallback = true; // def to true, if this is disabled m_gaussian2_fallback should be disabled too float m_gaussian1_strength = 1.45f; bool m_gaussian2_fallback = true; // def to true, hopefully rarely kicks in float m_gaussian2_strength = 1.83f; - + // m_disable_delta_endpoint_usage may give a slight increase in RDO ASTC encoding efficiency. It's also faster. bool m_disable_delta_endpoint_usage = false; @@ -67,7 +70,7 @@ namespace astc_6x6_hdr bool m_disable_twothree_subsets = false; // def to false bool m_use_solid_blocks = true; // def to true bool m_use_runs = true; // def to true - bool m_block_stat_optimizations_flag = true; // def to true + bool m_block_stat_optimizations_flag = true; // def to true bool m_rdo_candidate_diversity_boost = true; // def to true float m_rdo_candidate_diversity_boost_bit_window_weight = 1.2f; @@ -96,7 +99,7 @@ namespace astc_6x6_hdr basisu::fmt_debug_printf("m_rdo_candidate_diversity_boost: {}, m_rdo_candidate_diversity_boost_bit_window_weight: {}\n", m_rdo_candidate_diversity_boost, m_rdo_candidate_diversity_boost_bit_window_weight); basisu::fmt_debug_printf("m_favor_higher_compression: {}, m_num_reuse_xy_deltas: {}\n", m_favor_higher_compression, m_num_reuse_xy_deltas); } - + astc_hdr_6x6_global_config() { } @@ -121,7 +124,7 @@ namespace astc_6x6_hdr basisu::image_metrics m_im_bc6h_log2; basisu::image_metrics m_im_bc6h_half; }; - + // The input image should be unpadded to 6x6 boundaries, i.e. the original unexpanded image. bool compress_photo(const basisu::imagef& orig_src_img, const astc_hdr_6x6_global_config& global_cfg, basisu::job_pool* pJob_pool, basisu::uint8_vec& intermediate_tex_data, basisu::uint8_vec& astc_tex_data, result_metrics& metrics); diff --git a/encoder/basisu_astc_hdr_common.cpp b/encoder/basisu_astc_hdr_common.cpp index 65be44c..d4df682 100644 --- a/encoder/basisu_astc_hdr_common.cpp +++ b/encoder/basisu_astc_hdr_common.cpp @@ -1,4 +1,16 @@ // File: basisu_astc_hdr_common.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "basisu_enc.h" #include "basisu_gpu_texture.h" #include "../transcoder/basisu_astc_helpers.h" @@ -14,6 +26,7 @@ using namespace basist; namespace basisu { +// Beware: the first entry is the # of weight levels for that BISE range. const uint8_t g_ise_weight_lerps[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][33] = { { 2, 0, 64 }, // 0, note ise range=0 is invalid for 4x4 block sizes (<24 weight bits in the block) @@ -117,7 +130,7 @@ static void compute_half_to_qlog_table(uint32_t bits, uint16_t* pTable, const ba float best_err = BIG_FLOAT_VAL; uint32_t best_qlog = 0; - + double prev_err = BIG_FLOAT_VAL; // For all possible qlog's @@ -141,13 +154,13 @@ static void compute_half_to_qlog_table(uint32_t bits, uint16_t* pTable, const ba } prev_err = err; - + // Find best if (err < best_err) { best_err = err; best_qlog = i; - + if (best_err == 0.0f) break; } @@ -171,7 +184,7 @@ static void init_qlog_tables() #if BASISU_MULTITHREADED_INIT job_pool jp(3); - + for (uint32_t bits = HALF_TO_QLOG_TABS_MIN_BITS; bits <= HALF_TO_QLOG_TABS_MAX_BITS; bits++) { jp.add_job( [bits, &qlog16_to_float]() { compute_half_to_qlog_table(bits, g_pHalf_to_qlog_tabs[bits - HALF_TO_QLOG_TABS_MIN_BITS], qlog16_to_float); }); @@ -328,7 +341,7 @@ static bool compute_least_squares_endpoints_rgb( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec3F* pXl, vec3F* pXh, const vec4F* pColors, const aabb3F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; @@ -339,7 +352,7 @@ static bool compute_least_squares_endpoints_rgb( for (uint32_t i = 0; i < N; i++) { const uint32_t sel = pSelectors[i]; - + z00 += pSelector_weights[sel][0]; z10 += pSelector_weights[sel][1]; z11 += pSelector_weights[sel][2]; @@ -373,7 +386,7 @@ static bool compute_least_squares_endpoints_rgb( iz01 = -z01 * det; iz10 = -z10 * det; iz11 = z00 * det; - + (*pXl)[0] = (float)(iz00 * q00_r + iz01 * q10_r); (*pXh)[0] = (float)(iz10 * q00_r + iz11 * q10_r); @@ -392,7 +405,7 @@ static bool compute_least_squares_endpoints_rgb( l = input_box[0][c]; h = input_box[1][c]; } - + (*pXl)[c] = l; (*pXh)[c] = h; } @@ -429,17 +442,17 @@ static bool compute_least_squares_endpoints_rgb( } static bool compute_least_squares_endpoints_rgb_raw_weights( - uint32_t N, const uint8_t* pRaw_weights, + uint32_t N, const uint8_t* pRaw_weights, vec3F* pXl, vec3F* pXh, const vec4F* pColors, const aabb3F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; float q00_b = 0.0f, q10_b = 0.0f, t_b = 0.0f; - + for (uint32_t i = 0; i < N; i++) { const float wt = (float)pRaw_weights[i] * (1.0f / 64.0f); @@ -541,13 +554,13 @@ static bool compute_least_squares_endpoints_2D( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec2F* pXl, vec2F* pXh, const vec2F* pColors, const aabb2F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; float q00_r = 0.0f, q10_r = 0.0f, t_r = 0.0f; float q00_g = 0.0f, q10_g = 0.0f, t_g = 0.0f; - + for (uint32_t i = 0; i < N; i++) { const uint32_t sel = pSelectors[i]; @@ -599,7 +612,7 @@ static bool compute_least_squares_endpoints_2D( (*pXl)[c] = l; (*pXh)[c] = h; } - + pXl->clamp(0.0f, MAX_QLOG16_VAL); pXh->clamp(0.0f, MAX_QLOG16_VAL); @@ -610,7 +623,7 @@ static bool compute_least_squares_endpoints_1D( uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, vec1F* pXl, vec1F* pXh, const vec1F* pColors, const aabb1F& input_box) { - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // https://web.archive.org/web/20150319232457/http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. float z00 = 0.0f, z01 = 0.0f, z10 = 0.0f, z11 = 0.0f; @@ -668,10 +681,10 @@ static bool compute_least_squares_endpoints_1D( } static bool compute_weighted_least_squares_endpoints_rgb( - uint32_t N, + uint32_t N, const uint8_t* pSelectors, const vec4F* pSelector_weights, const float* pRaw_weights, /* ti */ const float* pEmphasis_weights /* wi */, - vec3F* pXl, vec3F* pXh, + vec3F* pXl, vec3F* pXh, const vec4F* pColors, /* pi */ const aabb3F& input_box) { @@ -702,7 +715,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( const float pi_r = pColors[i][0], pi_g = pColors[i][1], pi_b = pColors[i][2]; weighted_mean_tw += wi * ti; - + weighted_mean_pw[0] += wi * pi_r; weighted_mean_pw[1] += wi * pi_g; weighted_mean_pw[2] += wi * pi_b; @@ -722,7 +735,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( const float wi = pEmphasis_weights[i]; const float ti = pSelectors ? pSelector_weights[pSelectors[i]][3] : pRaw_weights[i]; const float pi_r = pColors[i][0], pi_g = pColors[i][1], pi_b = pColors[i][2]; - + spt[0] += wi * (pi_r - weighted_mean_pw[0]) * (ti - weighted_mean_tw); spt[1] += wi * (pi_g - weighted_mean_pw[1]) * (ti - weighted_mean_tw); spt[2] += wi * (pi_b - weighted_mean_pw[2]) * (ti - weighted_mean_tw); @@ -737,7 +750,7 @@ static bool compute_weighted_least_squares_endpoints_rgb( { float h = weighted_mean_pw[i] + (spt[i] / stt) * (1.0f - weighted_mean_tw); float l = weighted_mean_pw[i] - (spt[i] / stt) * weighted_mean_tw; - + (*pXh)[i] = h; (*pXl)[i] = l; } @@ -748,10 +761,10 @@ static bool compute_weighted_least_squares_endpoints_rgb( return true; } -static vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; +vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; -static uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index -static uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index +uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index +uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index static void encode_astc_hdr_init() { @@ -800,7 +813,7 @@ void astc_hdr_enc_init() astc_hdr_core_init(); - astc_helpers::init_tables(true); + astc_helpers::init_tables(); init_qlog_tables(); @@ -821,7 +834,7 @@ void interpolate_qlog12_colors( { for (uint32_t j = 0; j < 3; j++) { - assert(in_range(e[i][j], 0, 0xFFF)); + assert(is_in_range(e[i][j], 0, 0xFFF)); } } @@ -1128,7 +1141,7 @@ double eval_selectors( const vec3F low_color((float)pDecoded_half[lo_index * 3 + 0], (float)pDecoded_half[lo_index * 3 + 1], (float)pDecoded_half[lo_index * 3 + 2]); const vec3F high_color((float)pDecoded_half[hi_index * 3 + 0], (float)pDecoded_half[hi_index * 3 + 1], (float)pDecoded_half[hi_index * 3 + 2]); const vec3F mid_color((float)pDecoded_half[mid_index * 3 + 0], (float)pDecoded_half[mid_index * 3 + 1], (float)pDecoded_half[mid_index * 3 + 2]); - + const vec3F block_dir(high_color - low_color); for (uint32_t p = 0; p < num_pixels; p++) @@ -1140,11 +1153,11 @@ double eval_selectors( const int64_t desired_half_r_q = q2(desired_r, coptions.m_q_log_bias); const int64_t desired_half_g_q = q2(desired_g, coptions.m_q_log_bias); const int64_t desired_half_b_q = q2(desired_b, coptions.m_q_log_bias); - + // Determine which side of the middle plane the point is for a modest gain vec3F c((float)desired_r - mid_color[0], (float)desired_g - mid_color[1], (float)desired_b - mid_color[2]); float d = c.dot(block_dir); - + int i = 0, high_index = (num_weight_levels / 2) + 1; if (d >= 0.0f) { @@ -1240,11 +1253,11 @@ double eval_selectors_dual_plane( const uint32_t first_channel = (channel_index + 1) % 3; const uint32_t second_channel = (channel_index + 2) % 3; - + // First plane const double first_channel_weight = channel_weights[first_channel]; const double second_channel_weight = channel_weights[second_channel]; - + for (uint32_t p = 0; p < num_pixels; p++) { const half_float* pDesired_half = &pBlock_pixels_half[p * 3]; @@ -1284,7 +1297,7 @@ double eval_selectors_dual_plane( const half_float* pDesired_half = &pBlock_pixels_half[p * 3]; const double desired_half_a_q = q(pDesired_half[channel_index], coptions.m_q_log_bias); - + double lowest_e = BIG_FLOAT_VAL; // this is an approximation of MSLE @@ -1664,7 +1677,7 @@ bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2 v4 |= (best_vd0 & 31); v5 |= (best_vd1 & 31); - assert(in_range(v0, 0, 255) && in_range(v1, 0, 255) && in_range(v2, 0, 255) && in_range(v3, 0, 255) && in_range(v4, 0, 255) && in_range(v5, 0, 255)); + assert(is_in_range(v0, 0, 255) && is_in_range(v1, 0, 255) && is_in_range(v2, 0, 255) && is_in_range(v3, 0, 255) && is_in_range(v4, 0, 255) && is_in_range(v5, 0, 255)); pEndpoints[0] = (uint8_t)v0; pEndpoints[1] = (uint8_t)v1; @@ -1711,7 +1724,7 @@ bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2 bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& low_q16, const vec3F& high_q16, int& max_clamp_mag, bool early_out_if_clamped, int max_clamp_mag_accept_thresh) { assert(submode <= 7); - + const uint32_t a_bits = 9 + (submode >> 1); const int max_a_val = (1 << a_bits) - 1; @@ -1803,7 +1816,7 @@ void pack_astc_mode11_direct(uint8_t* pEndpoints, vec3F l_q16, vec3F h_q16) // this quantizes R and G as 7 bits vs. 8, for grayscale. //l_q = g_half_to_qlog7[bounds_check((uint32_t)l_half, 0U, 32768U)] << 1; //h_q = g_half_to_qlog7[bounds_check((uint32_t)h_half, 0U, 32768U)] << 1; - + l_q = minimum(l_q, MAX_QLOG8); h_q = minimum(h_q, MAX_QLOG8); } @@ -1982,7 +1995,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[0], 8); // R8 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[0], 10); // R10 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -1996,7 +2009,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[0], 10); // R10 x6 = get_bit(qlog[0], 9); // R9 break; @@ -2010,7 +2023,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[0], 8); // R8 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[0], 6); // R6 - x4 = get_bit(qlog[3], 7); // S7 + x4 = get_bit(qlog[3], 7); // S7 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2024,7 +2037,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[0], 7); // R7 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2039,7 +2052,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[2], 6); // B6 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[0], 7); // R7 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2052,7 +2065,7 @@ bool pack_astc_mode7_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& x1 = get_bit(qlog[1], 5); // G5 x2 = get_bit(qlog[2], 6); // B6 x3 = get_bit(qlog[2], 5); // B5 - x4 = get_bit(qlog[0], 6); // R6 + x4 = get_bit(qlog[0], 6); // R6 x5 = get_bit(qlog[3], 6); // S6 x6 = get_bit(qlog[3], 5); // S5 break; @@ -2143,7 +2156,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) pEndpoints[1] = (uint8_t)desc.m_b1; pEndpoints[3] = (uint8_t)desc.m_d0; pEndpoints[5] = (uint8_t)desc.m_d1 | 128; - + return true; } @@ -2161,9 +2174,9 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) return false; const int va = desc.m_a, vb0 = desc.m_b0, vb1 = desc.m_b1, vc = desc.m_c, vd0 = desc.m_d0, vd1 = desc.m_d1; - + int v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0, v5 = 0; - + int x0 = 0, x1 = 0, x2 = 0, x3 = 0, x4 = 0, x5 = 0; switch (desc.m_submode) { @@ -2222,7 +2235,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) v4 |= (vd0 & 31); v5 |= (vd1 & 31); - assert(in_range(v0, 0, 255) && in_range(v1, 0, 255) && in_range(v2, 0, 255) && in_range(v3, 0, 255) && in_range(v4, 0, 255) && in_range(v5, 0, 255)); + assert(is_in_range(v0, 0, 255) && is_in_range(v1, 0, 255) && is_in_range(v2, 0, 255) && is_in_range(v3, 0, 255) && is_in_range(v4, 0, 255) && is_in_range(v5, 0, 255)); pEndpoints[0] = (uint8_t)v0; pEndpoints[1] = (uint8_t)v1; @@ -2236,7 +2249,7 @@ bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints) static inline int astc_hdr_sign_extend(int src, int num_src_bits) { - assert(basisu::in_range(num_src_bits, 2, 31)); + assert(basisu::is_in_range(num_src_bits, 2, 31)); const bool negative = (src & (1 << (num_src_bits - 1))) != 0; if (negative) @@ -2261,7 +2274,7 @@ void unpack_mode11(const uint8_t* pEndpoints, mode11_log_desc& desc) desc.m_b1 = pEndpoints[1]; desc.m_d0 = pEndpoints[3]; desc.m_d1 = pEndpoints[5] & 0x7F; - + return; } @@ -2384,7 +2397,7 @@ void decode_cem_7_config(const uint8_t* pEndpoints, int& submode_index, int &maj bool pack_mode11( const vec3F& low_color_q16, const vec3F& high_color_q16, - uint32_t ise_endpoint_range, uint8_t* pEndpoints, + uint32_t ise_endpoint_range, uint8_t* pEndpoints, const astc_hdr_codec_base_options& coptions, bool direct_only, int32_t first_submode, int32_t last_submode, bool ignore_clamping, uint32_t& submode_used) { @@ -2521,7 +2534,7 @@ bool pack_mode11( } // d } // c } // if (coptions.m_ultra_quant) - + submode_used = best_submode + 1; return (best_trial_dist != BIG_FLOAT_VAL); @@ -3039,7 +3052,7 @@ bool try_mode7( clear_obj(best_trial_endpoints); double best_trial_dist = BIG_FLOAT_VAL; int best_trial_submode = 0; - + for (int submode = first_submode; submode <= last_submode; submode++) { const int MAX_CLAMP_MAG_ACCEPT_THRESH = 16; @@ -3216,6 +3229,8 @@ double encode_astc_hdr_block_mode_11( float l = BIG_FLOAT_VAL, h = -BIG_FLOAT_VAL; vec3F low_color_q16, high_color_q16; + low_color_q16.clear(); + high_color_q16.clear(); for (uint32_t i = 0; i < num_pixels; i++) { @@ -3236,9 +3251,9 @@ double encode_astc_hdr_block_mode_11( high_color_q16 = pBlock_pixels_q16[i]; } } - + vec3F old_low_color_q16(low_color_q16), old_high_color_q16(high_color_q16); - + for (uint32_t i = 0; i < 3; i++) { low_color_q16[i] = lerp(old_low_color_q16[i], old_high_color_q16[i], 1.0f / 64.0f); @@ -3253,7 +3268,7 @@ double encode_astc_hdr_block_mode_11( clear_obj(trial_blk_weights); double trial_blk_error = BIG_FLOAT_VAL; - + bool did_improve = try_mode11(num_pixels, trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_best_submode, low_color_q16, high_color_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, @@ -3330,7 +3345,7 @@ double encode_astc_hdr_block_mode_11( if (!compute_least_squares_endpoints_rgb(num_pixels, trial_blk_weights, &g_astc_ls_weights_ise[ise_weight_range][0], &l_q16, &h_q16, pBlock_pixels_q16, color_box_q16)) break; - + bool was_improved = try_mode11(num_pixels, blk_endpoints, blk_weights, cur_block_error, best_submode, l_q16, h_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, @@ -3359,7 +3374,7 @@ double encode_astc_hdr_block_mode_11( float lw = LOW_EMPHASIS_WEIGHT, mw = MIDDLE_EMPHASIS_WEIGHT, hw = HIGH_EMPHASIS_WEIGHT; if (opt_mode == cWeightedLeastSquaresHeavy) lw = LOW_EMPHASIS_WEIGHT_HEAVY, mw = MIDDLE_EMPHASIS_WEIGHT_HEAVY, hw = HIGH_EMPHASIS_WEIGHT_HEAVY; - + for (uint32_t i = 0; i < num_pixels; i++) { vec3F k(vec3F(pBlock_pixels_q16[i]) - block_mean_color_q16); @@ -3368,7 +3383,7 @@ double encode_astc_hdr_block_mode_11( assert((kd >= l) && (kd <= h)); float v = (kd - l) / (h - l); - + if (v < mid) v = lerp(lw, mw, v / mid); else @@ -3617,6 +3632,8 @@ double encode_astc_hdr_block_downsampled_mode_11( float l = BIG_FLOAT_VAL, h = -BIG_FLOAT_VAL; vec3F low_color_q16, high_color_q16; + low_color_q16.clear(); + high_color_q16.clear(); for (uint32_t i = 0; i < num_pixels; i++) { @@ -3655,7 +3672,7 @@ double encode_astc_hdr_block_downsampled_mode_11( clear_obj(trial_blk_endpoints); clear_obj(trial_blk_weights); - + double trial_blk_error = BIG_FLOAT_VAL; bool could_pack = try_mode11(num_pixels, trial_blk_endpoints, trial_blk_weights, trial_blk_error, trial_best_submode, @@ -3696,7 +3713,7 @@ double encode_astc_hdr_block_downsampled_mode_11( } else if (pass) break; - + if ((opt_mode == cWeightedLeastSquares) || (opt_mode == cWeightedLeastSquaresHeavy)) { float emphasis_weights[MAX_ASTC_HDR_ENC_BLOCK_PIXELS]; @@ -3797,7 +3814,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( assert((first_submode >= FIRST_MODE11_SUBMODE_INDEX) && (first_submode <= last_submode)); assert(last_submode <= MAX_MODE11_SUBMODE_INDEX); - + assert(num_pixels <= MAX_ASTC_HDR_ENC_BLOCK_PIXELS); best_submode = 0; @@ -3873,7 +3890,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( double trial_blk_error = BIG_FLOAT_VAL; bool did_improve = try_mode11_dual_plane(channel_index, num_pixels, trial_blk_endpoints, trial_blk_weights0, trial_blk_weights1, trial_blk_error, trial_best_submode, - low_color_q16, high_color_q16, + low_color_q16, high_color_q16, pBlock_pixels_half, num_weight_levels, ise_weight_range, coptions, direct_only, ise_endpoint_range, constrain_ise_weight_selectors, first_submode, last_submode, ignore_clamping); @@ -3947,7 +3964,7 @@ double encode_astc_hdr_block_mode_11_dual_plane( memcpy(trial_blk_weights1, blk_weights1, num_pixels); } // pass - + return cur_block_error; } @@ -3962,7 +3979,7 @@ double encode_astc_hdr_block_mode_7( uint8_t* blk_endpoints, //[4] uint8_t* blk_weights, // [num_pixels] const astc_hdr_codec_base_options& coptions, - uint32_t ise_endpoint_range, + uint32_t ise_endpoint_range, int first_submode, int last_submode, const encode_astc_block_stats* pBlock_stats) { @@ -4008,7 +4025,7 @@ double encode_astc_hdr_block_mode_7( vec3F diff(high_color_q16 - low_color_q16); - // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, + // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, // i.e. it's equivalent to a vector of (1,1,1) multiplied by scale before the sub. We want to actually move along the grayscale axis, or (0.577350259, 0.577350259, 0.577350259). float s_q16 = diff.dot(block_axis_q16) * block_axis_q16[0]; @@ -4081,7 +4098,7 @@ double encode_astc_hdr_block_mode_7( vec3F alt_diff(alt_high_color_q16 - alt_low_color_q16); - // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, + // The mul here (* block_axis_q16[0]) is because the "S" or scale value is subtracted from the high color with a scale of 1.0, // i.e. it's equivalent to a vector of (1,1,1) multiplied by scale before the sub. We want to actually move along the grayscale axis, or (0.577350259, 0.577350259, 0.577350259). float alt_s_q16 = alt_diff.dot(block_axis_q16) * block_axis_q16[0]; @@ -4217,6 +4234,7 @@ void dequantize_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t } //-------------------------------------------------------------------------------------------------------------------------- +// Precomputed matrices via SLSQP (Sequential Least-Squares Quadratic Programming - scipy.optimize.minimize). Sharper results vs. other methods (like adjoint). // For each output (2x2) sample, the weight of each input (6x6) sample. static const float g_weight_downsample_6x6_to_2x2[4][36] = { @@ -4720,11 +4738,14 @@ static const float g_weight_downsample_6x6_to_6x6[36][36] = { //-------------------------------------------------------------------------------------------------------------------------- -const struct downsample_matrix_6x6 +struct downsample_matrix { uint32_t m_grid_width, m_grid_height; const float* m_p; -} g_downsample_matrices_6x6[] = { +}; + +downsample_matrix g_downsample_matrices_6x6[] = +{ { 2, 2, (const float*)g_weight_downsample_6x6_to_2x2 }, { 3, 2, (const float*)g_weight_downsample_6x6_to_3x2 }, { 4, 2, (const float*)g_weight_downsample_6x6_to_4x2 }, @@ -4753,11 +4774,892 @@ const struct downsample_matrix_6x6 }; //const uint32_t NUM_DOWNSAMPLE_MATRICES_6x6 = sizeof(g_downsample_matrices_6x6) / sizeof(g_downsample_matrices_6x6[0]); +//-------------------------------------------------------------------------------------------------------------------------- +// +// For each output (2x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x2[4][48] = { +{0.137431f, 0.119592f, 0.085575f, 0.056401f, 0.030751f, 0.000000f, 0.000000f, 0.000000f, 0.108851f, 0.086312f, 0.064884f, 0.039119f, 0.027653f, 0.000000f, 0.000000f, 0.000000f, 0.073703f, 0.067584f, 0.045034f, 0.032697f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.033828f, 0.058911f, 0.081870f, 0.120975f, 0.137384f, 0.000000f, 0.000000f, 0.000000f, 0.026912f, 0.038126f, 0.065247f, 0.083628f, 0.109730f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037909f, 0.044325f, 0.065160f, 0.074043f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021952f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024645f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074133f, 0.065243f, 0.043065f, 0.035114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105931f, 0.087385f, 0.065848f, 0.035699f, 0.030068f, 0.000000f, 0.000000f, 0.000000f, 0.136321f, 0.121324f, 0.086171f, 0.057503f, 0.031553f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037022f, 0.042379f, 0.063662f, 0.075871f, 0.000000f, 0.000000f, 0.000000f, 0.031315f, 0.037129f, 0.065785f, 0.084055f, 0.107841f, 0.000000f, 0.000000f, 0.000000f, 0.030537f, 0.057932f, 0.086040f, 0.120055f, 0.136127f}, +}; + +// For each output (3x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x2[6][48] = { +{0.212556f, 0.137038f, 0.067006f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.172663f, 0.105023f, 0.058944f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113989f, 0.074111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037147f, 0.021524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.077366f, 0.142656f, 0.145067f, 0.074900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048644f, 0.106713f, 0.104141f, 0.052434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048972f, 0.079367f, 0.079508f, 0.040229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064479f, 0.139823f, 0.212207f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053987f, 0.104596f, 0.171728f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026564f, 0.071759f, 0.119334f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.115689f, 0.072510f, 0.021389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170967f, 0.106096f, 0.061696f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210888f, 0.137969f, 0.065274f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045147f, 0.080905f, 0.078591f, 0.043486f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045421f, 0.106778f, 0.106427f, 0.050794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.079169f, 0.139959f, 0.144180f, 0.079143f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033940f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021724f, 0.070791f, 0.117496f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059938f, 0.109787f, 0.170583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064517f, 0.139526f, 0.211698f}, +}; + +// For each output (4x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x2[8][48] = { +{0.275657f, 0.133248f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.225305f, 0.089819f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.147466f, 0.079439f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049065f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.071558f, 0.188360f, 0.141460f, 0.027429f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068719f, 0.139588f, 0.107851f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024602f, 0.112032f, 0.076880f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019401f, 0.000000f, 0.022120f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.025244f, 0.140416f, 0.189606f, 0.065541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021281f, 0.106671f, 0.142270f, 0.062848f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068039f, 0.102306f, 0.026541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023517f, 0.025720f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136533f, 0.275463f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086827f, 0.223674f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077361f, 0.153684f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048293f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149189f, 0.077647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.222753f, 0.093443f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.273639f, 0.135036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022695f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027966f, 0.116923f, 0.074704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066610f, 0.140552f, 0.119791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070250f, 0.192769f, 0.140414f, 0.027327f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026026f, 0.032280f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073723f, 0.105102f, 0.027631f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113307f, 0.139466f, 0.059915f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027161f, 0.140907f, 0.189935f, 0.064546f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045275f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074412f, 0.151685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.094074f, 0.223897f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136604f, 0.274053f}, +}; + +// For each output (5x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x2[10][48] = { +{0.298257f, 0.099048f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.242705f, 0.083012f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155959f, 0.035340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054463f, 0.031217f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.149629f, 0.250491f, 0.037003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113317f, 0.192720f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.093738f, 0.138010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025093f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.193314f, 0.196494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.163178f, 0.158983f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.112334f, 0.115733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028572f, 0.031390f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028975f, 0.256222f, 0.142262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191874f, 0.111703f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.137754f, 0.096234f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105369f, 0.297279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.081692f, 0.239675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031939f, 0.162333f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031404f, 0.050308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053972f, 0.028379f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158432f, 0.035219f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.238959f, 0.089734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.294641f, 0.100664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.090008f, 0.147020f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.103221f, 0.190008f, 0.024843f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.139784f, 0.245082f, 0.025860f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032527f, 0.032618f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117780f, 0.108323f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155910f, 0.159880f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197210f, 0.195753f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.042681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.138684f, 0.099059f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.186926f, 0.105714f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029545f, 0.254477f, 0.142915f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029953f, 0.051219f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029174f, 0.163463f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087461f, 0.240531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.103819f, 0.294380f}, +}; + +// For each output (6x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x2[12][48] = { +{0.362153f, 0.050427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.296074f, 0.031598f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.192551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067197f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.240020f, 0.169624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196469f, 0.128913f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.131714f, 0.098049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035210f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.105361f, 0.301218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086270f, 0.220336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047552f, 0.171037f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022966f, 0.045259f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.287211f, 0.111854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.224383f, 0.097742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167408f, 0.037607f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036827f, 0.036969f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152162f, 0.235841f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108280f, 0.202388f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091687f, 0.151852f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051343f, 0.374208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.304381f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064793f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.290484f, 0.038424f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357650f, 0.055589f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035640f, 0.019558f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.133571f, 0.100435f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184400f, 0.125111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228117f, 0.173168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043438f, 0.175074f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089766f, 0.235789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108452f, 0.302770f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037495f, 0.032008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.168503f, 0.033572f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.226763f, 0.101709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.292934f, 0.107016f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019003f, 0.018791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100854f, 0.125828f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107572f, 0.206978f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169736f, 0.251237f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024678f, 0.204824f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.301594f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040204f, 0.368158f}, +}; + +// For each output (7x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x2[14][48] = { +{0.396534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324924f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210380f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.365804f, 0.047637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.288211f, 0.031570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.215416f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051362f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.277573f, 0.121338f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219048f, 0.084370f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023178f, 0.000000f, 0.161469f, 0.031346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034866f, 0.046814f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.194115f, 0.218789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.163854f, 0.137782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020281f, 0.000000f, 0.127129f, 0.138049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089911f, 0.279003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100285f, 0.229490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026109f, 0.164969f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036219f, 0.074014f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033369f, 0.385493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.300028f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.222803f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320906f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218670f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064618f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064591f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.213009f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324054f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.398346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052403f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.280900f, 0.028228f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364696f, 0.054830f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040226f, 0.027986f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.172678f, 0.019447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228976f, 0.118935f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278251f, 0.113500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.017206f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022373f, 0.000000f, 0.138786f, 0.130317f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024343f, 0.000000f, 0.127713f, 0.134415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187440f, 0.195205f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033347f, 0.041046f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029210f, 0.133093f, 0.000000f, 0.020285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.102427f, 0.246296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104431f, 0.289864f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027153f, 0.048478f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032573f, 0.217822f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022617f, 0.372424f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061793f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394594f}, +}; + +// For each output (8x2) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x2[16][48] = { +{0.397679f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325539f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.208885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067897f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.394986f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.323551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218305f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063158f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.400685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325867f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059075f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.398573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319207f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212413f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069808f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.323398f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.404990f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207631f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065371f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396891f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212780f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396345f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.321731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064801f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324204f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.398456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.221286f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319039f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064375f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.221627f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067161f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214405f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322795f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395638f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065100f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.209382f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.325769f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.399749f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072177f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.207268f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.318619f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401935f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063557f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.316546f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.402413f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061762f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218082f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324604f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395552f}, +}; + +// For each output (2x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x3[6][48] = { +{0.205910f, 0.181220f, 0.131230f, 0.084091f, 0.045598f, 0.000000f, 0.000000f, 0.000000f, 0.115248f, 0.106195f, 0.073083f, 0.057425f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.054674f, 0.092055f, 0.125587f, 0.176378f, 0.202284f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055452f, 0.075306f, 0.102574f, 0.115689f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044070f, 0.029520f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136903f, 0.115512f, 0.084403f, 0.050846f, 0.035490f, 0.000000f, 0.000000f, 0.000000f, 0.143459f, 0.115683f, 0.085020f, 0.053056f, 0.036572f, 0.000000f, 0.000000f, 0.000000f, 0.043466f, 0.026000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025190f, 0.040099f, 0.000000f, 0.000000f, 0.000000f, 0.037965f, 0.050927f, 0.083471f, 0.112563f, 0.137468f, 0.000000f, 0.000000f, 0.000000f, 0.033927f, 0.046348f, 0.085573f, 0.114643f, 0.134372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024810f, 0.028641f, 0.044003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111326f, 0.107232f, 0.073233f, 0.050676f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204047f, 0.179532f, 0.131819f, 0.088809f, 0.053325f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023277f, 0.054224f, 0.067723f, 0.100097f, 0.113199f, 0.000000f, 0.000000f, 0.000000f, 0.047881f, 0.085543f, 0.130088f, 0.176198f, 0.201769f}, +}; + +// For each output (3x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x3[9][48] = { +{0.327238f, 0.215195f, 0.108640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184524f, 0.118385f, 0.046018f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.109423f, 0.206952f, 0.207632f, 0.108494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064973f, 0.120899f, 0.114663f, 0.066964f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107663f, 0.213426f, 0.326644f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045643f, 0.119988f, 0.186636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060005f, 0.030140f, 0.020392f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193258f, 0.127396f, 0.061395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196600f, 0.132656f, 0.063337f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060793f, 0.029915f, 0.024113f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032682f, 0.042599f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070428f, 0.145040f, 0.144782f, 0.074883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069308f, 0.145612f, 0.133265f, 0.071190f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035901f, 0.034311f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030350f, 0.056939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060846f, 0.125850f, 0.201518f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063906f, 0.129434f, 0.203119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035006f, 0.026673f, 0.066360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184897f, 0.119434f, 0.045977f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.328093f, 0.217057f, 0.104542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064974f, 0.120280f, 0.118724f, 0.069494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111457f, 0.199814f, 0.204785f, 0.110472f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038193f, 0.124885f, 0.182125f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105011f, 0.218548f, 0.331237f}, +}; + +// For each output (4x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x3[12][48] = { +{0.424820f, 0.213734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.237540f, 0.123907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.101064f, 0.293828f, 0.214193f, 0.045263f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051229f, 0.170008f, 0.124414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.043452f, 0.216897f, 0.293802f, 0.110908f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114842f, 0.173267f, 0.046832f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204747f, 0.427412f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.126209f, 0.241633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087490f, 0.023647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277233f, 0.116842f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.282751f, 0.124394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087642f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024375f, 0.043221f, 0.025504f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075199f, 0.165822f, 0.130107f, 0.031544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074010f, 0.171441f, 0.131257f, 0.016920f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037357f, 0.043775f, 0.029468f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034358f, 0.046676f, 0.025003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026567f, 0.127081f, 0.172282f, 0.077309f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028046f, 0.132256f, 0.162992f, 0.075728f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033213f, 0.036679f, 0.021810f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.083610f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116623f, 0.293550f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.118246f, 0.292686f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.234002f, 0.132935f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.422801f, 0.210262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037740f, 0.173712f, 0.127636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107054f, 0.296425f, 0.213343f, 0.044090f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.122782f, 0.174732f, 0.044321f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046279f, 0.214323f, 0.289278f, 0.108285f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.125079f, 0.236461f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.208583f, 0.429877f}, +}; + +// For each output (5x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x3[15][48] = { +{0.490219f, 0.168976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.273361f, 0.067444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.213329f, 0.380538f, 0.048722f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.138224f, 0.219188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.309867f, 0.312289f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.189101f, 0.188743f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037522f, 0.380550f, 0.216834f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.225818f, 0.139276f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.164462f, 0.488476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072635f, 0.274427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085550f, 0.041856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277218f, 0.100778f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.279523f, 0.102655f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086943f, 0.025474f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018474f, 0.000000f, 0.000000f, 0.023807f, 0.063654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142638f, 0.245307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.145790f, 0.254064f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047600f, 0.058666f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047090f, 0.051660f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197880f, 0.207261f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.205538f, 0.186457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052816f, 0.051298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018852f, 0.055366f, 0.033613f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247747f, 0.138008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030549f, 0.240788f, 0.147930f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066598f, 0.020549f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031861f, 0.081013f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095562f, 0.286515f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091897f, 0.287997f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038590f, 0.086564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.268683f, 0.083034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.485628f, 0.162655f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.121869f, 0.229484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.218817f, 0.384593f, 0.045237f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.182342f, 0.183530f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320205f, 0.313923f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.217960f, 0.138650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051048f, 0.375126f, 0.217217f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064150f, 0.273673f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169346f, 0.492831f}, +}; + +// For each output (6x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x3[18][48] = { +{0.567729f, 0.085252f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.316321f, 0.030698f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.359927f, 0.264711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204426f, 0.170936f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.160854f, 0.493683f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055911f, 0.289551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.471204f, 0.180222f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.281132f, 0.067442f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.244512f, 0.369052f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158920f, 0.227515f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066465f, 0.597036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.336500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104579f, 0.023148f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.338908f, 0.039468f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.344319f, 0.042826f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059448f, 0.022978f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245888f, 0.156583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.251094f, 0.164427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073868f, 0.025715f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047831f, 0.060057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116572f, 0.271105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108894f, 0.276085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039515f, 0.079942f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.080438f, 0.048264f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267123f, 0.113138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.263081f, 0.110654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077711f, 0.039591f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020193f, 0.059109f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154371f, 0.249388f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148917f, 0.263084f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021121f, 0.083817f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024900f, 0.107003f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.375065f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.378856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114175f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.311342f, 0.043011f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.565421f, 0.080225f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.192162f, 0.168731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354606f, 0.265733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069515f, 0.282839f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.159765f, 0.487881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278646f, 0.072312f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.480532f, 0.168510f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.157488f, 0.194745f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.261639f, 0.386129f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043524f, 0.320675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055191f, 0.580610f}, +}; + +// For each output (7x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x3[21][48] = { +{0.641452f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.358548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.571435f, 0.068076f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.330216f, 0.030272f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.442607f, 0.191771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243785f, 0.063036f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018329f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019157f, 0.000000f, 0.021315f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.273064f, 0.307420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195541f, 0.177034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022294f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024647f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.151030f, 0.456644f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.078617f, 0.291813f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060980f, 0.596856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.342163f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.639429f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.360571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.378786f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.118726f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.090755f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356378f, 0.041502f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359468f, 0.040845f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.091221f, 0.019830f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.078340f, 0.030772f, 0.000000f, 0.017555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267597f, 0.100863f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.271447f, 0.100798f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064330f, 0.068296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044982f, 0.034940f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021793f, 0.000000f, 0.194246f, 0.216278f, 0.000000f, 0.022234f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.203237f, 0.184740f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019217f, 0.018086f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016776f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047044f, 0.060726f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086110f, 0.270497f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100587f, 0.267194f, 0.000000f, 0.020092f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050739f, 0.097011f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023976f, 0.094747f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036130f, 0.353791f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032724f, 0.369552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089080f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.386732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390932f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114916f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354042f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.645958f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.337170f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589668f, 0.073162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.281005f, 0.071771f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.450506f, 0.196718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021998f, 0.000000f, 0.000000f, 0.025261f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032091f, 0.000000f, 0.182952f, 0.186377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.270805f, 0.280517f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020667f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064614f, 0.248064f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.182212f, 0.484444f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046780f, 0.341462f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041817f, 0.569940f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.355095f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.644905f}, +}; + +// For each output (8x3) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x3[24][48] = { +{0.642405f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357595f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.643957f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356043f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.642833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357167f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.637580f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.362420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.642714f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.357286f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.637481f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.362519f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646282f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.640587f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359413f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.379885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116950f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104449f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396859f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400104f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.098588f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.102359f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394242f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101667f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.096440f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392155f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400404f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.111000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114593f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.382704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.112742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109021f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.388517f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105580f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.108474f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401518f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.100446f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106886f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387604f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392295f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.113215f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646427f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643079f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.363744f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.636256f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.356177f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.354225f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.645775f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.359749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.640251f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364443f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.635557f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.353912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.646088f}, +}; + +// For each output (2x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x4[8][48] = { +{0.266475f, 0.237248f, 0.170961f, 0.108932f, 0.059980f, 0.000000f, 0.000000f, 0.000000f, 0.069153f, 0.052080f, 0.035172f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.071584f, 0.118291f, 0.158003f, 0.229344f, 0.262308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040608f, 0.047117f, 0.072745f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.133546f, 0.123736f, 0.085634f, 0.071146f, 0.020522f, 0.000000f, 0.000000f, 0.000000f, 0.181365f, 0.152470f, 0.109189f, 0.071277f, 0.051114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068769f, 0.083081f, 0.122611f, 0.135462f, 0.000000f, 0.000000f, 0.000000f, 0.052661f, 0.073804f, 0.122675f, 0.158233f, 0.182705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185771f, 0.157833f, 0.115265f, 0.071389f, 0.049909f, 0.000000f, 0.000000f, 0.000000f, 0.134315f, 0.122577f, 0.090159f, 0.072782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049580f, 0.068443f, 0.120275f, 0.155720f, 0.183091f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072223f, 0.092680f, 0.123123f, 0.134866f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061367f, 0.051211f, 0.034360f, 0.000000f, 0.028160f, 0.000000f, 0.000000f, 0.000000f, 0.255536f, 0.224675f, 0.167736f, 0.113503f, 0.063453f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033855f, 0.000000f, 0.030092f, 0.044250f, 0.067673f, 0.000000f, 0.000000f, 0.000000f, 0.059731f, 0.111955f, 0.169044f, 0.224131f, 0.259268f}, +}; + +// For each output (3x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x4[12][48] = { +{0.405143f, 0.264455f, 0.127900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105076f, 0.051679f, 0.045747f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.025952f, 0.148689f, 0.283429f, 0.283899f, 0.145415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061558f, 0.051058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.124702f, 0.268998f, 0.405480f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043101f, 0.052379f, 0.105340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214261f, 0.145181f, 0.047508f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.296952f, 0.196156f, 0.099941f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084673f, 0.137735f, 0.144414f, 0.077484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086806f, 0.178074f, 0.179109f, 0.089543f, 0.022161f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050723f, 0.149013f, 0.214357f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101549f, 0.190388f, 0.293970f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.293440f, 0.200404f, 0.104808f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212205f, 0.141684f, 0.047458f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085757f, 0.179609f, 0.175648f, 0.084745f, 0.021210f, 0.000000f, 0.000000f, 0.000000f, 0.083231f, 0.140659f, 0.147264f, 0.081878f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.104715f, 0.195444f, 0.297105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052478f, 0.135662f, 0.214595f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.105858f, 0.047177f, 0.044681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407919f, 0.269431f, 0.124933f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066066f, 0.061881f, 0.023069f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149307f, 0.272481f, 0.277246f, 0.149950f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036865f, 0.065377f, 0.096438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.123758f, 0.269301f, 0.408262f}, +}; + +// For each output (4x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x4[16][48] = { +{0.550981f, 0.273527f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.143555f, 0.031938f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.122629f, 0.360487f, 0.261668f, 0.049773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061033f, 0.081604f, 0.062805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.049839f, 0.269578f, 0.365997f, 0.133966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.048352f, 0.083803f, 0.048464f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.267525f, 0.553972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034129f, 0.144375f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.277118f, 0.159322f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390449f, 0.173111f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047384f, 0.191890f, 0.131656f, 0.024565f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109738f, 0.256529f, 0.192107f, 0.046132f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031695f, 0.141682f, 0.193059f, 0.054775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036195f, 0.182374f, 0.246275f, 0.113945f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.160040f, 0.281798f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.166904f, 0.391257f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392178f, 0.179451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.279598f, 0.148773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107261f, 0.247609f, 0.198942f, 0.036907f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054678f, 0.195067f, 0.134127f, 0.025410f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017019f, 0.017319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032887f, 0.182133f, 0.239063f, 0.107658f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026552f, 0.139058f, 0.187193f, 0.051118f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.169923f, 0.395389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148923f, 0.285765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142165f, 0.038534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.547445f, 0.271856f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044944f, 0.076529f, 0.068448f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.125039f, 0.368874f, 0.262015f, 0.054151f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.059929f, 0.083064f, 0.044633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053433f, 0.265593f, 0.362429f, 0.130919f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045972f, 0.135681f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.264414f, 0.553933f}, +}; + +// For each output (5x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x4[20][48] = { +{0.596845f, 0.198746f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.148428f, 0.055981f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.278053f, 0.491329f, 0.050522f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064229f, 0.115868f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.404918f, 0.399709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.097883f, 0.097489f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050295f, 0.498737f, 0.280436f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117869f, 0.052664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200415f, 0.589668f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063856f, 0.146061f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.306027f, 0.097934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.428737f, 0.167302f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.155850f, 0.258285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187173f, 0.344891f, 0.035315f, 0.000000f, 0.018485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212411f, 0.213232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.283532f, 0.290826f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022380f, 0.255191f, 0.169763f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020378f, 0.342025f, 0.190264f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089095f, 0.316913f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.159089f, 0.434903f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.436982f, 0.169707f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.310539f, 0.082773f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.187439f, 0.337224f, 0.031428f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167442f, 0.252995f, 0.023472f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.298614f, 0.285810f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206405f, 0.209172f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033200f, 0.325724f, 0.185761f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030366f, 0.251622f, 0.153784f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.161862f, 0.437691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086681f, 0.313765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.149673f, 0.068654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589414f, 0.192260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038852f, 0.121054f, 0.025391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.280331f, 0.492424f, 0.041948f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.095308f, 0.102698f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407796f, 0.394198f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106939f, 0.057645f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058299f, 0.489157f, 0.287960f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063501f, 0.142763f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196593f, 0.597142f}, +}; + +// For each output (6x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x4[24][48] = { +{0.723801f, 0.094637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.181562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.476584f, 0.344817f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.116143f, 0.062457f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.194537f, 0.608409f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061561f, 0.135493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579284f, 0.209203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.135477f, 0.076035f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.308340f, 0.460085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052476f, 0.139411f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019970f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019719f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.082209f, 0.732181f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.358932f, 0.060659f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.503915f, 0.076494f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.237301f, 0.199098f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.332364f, 0.231237f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088364f, 0.322995f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.173711f, 0.414930f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.312366f, 0.093336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.392413f, 0.164056f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019281f, 0.018548f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.178453f, 0.229682f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.214423f, 0.359860f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071976f, 0.390475f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.537548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.515147f, 0.078582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.364623f, 0.041649f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.337054f, 0.220008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249141f, 0.193797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.168802f, 0.423188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084285f, 0.323725f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411061f, 0.182411f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.329651f, 0.076877f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193953f, 0.352033f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.188543f, 0.265471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050266f, 0.555034f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.394700f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179003f, 0.029987f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.700087f, 0.090924f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019171f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.099147f, 0.059028f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.470203f, 0.352451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075527f, 0.135452f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.184084f, 0.604937f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136189f, 0.084874f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.576900f, 0.202037f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041868f, 0.099347f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.343377f, 0.515408f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044581f, 0.169532f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.062013f, 0.723875f}, +}; + +// For each output (7x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x4[28][48] = { +{0.798509f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201491f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.716711f, 0.085583f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167498f, 0.030208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.538182f, 0.218008f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114187f, 0.070138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020226f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020777f, 0.000000f, 0.000000f, 0.018482f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.367283f, 0.403492f, 0.000000f, 0.017972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071839f, 0.050645f, 0.000000f, 0.023445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020007f, 0.000000f, 0.000000f, 0.000000f, 0.022030f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023286f, 0.000000f, 0.000000f}, +{0.000000f, 0.026415f, 0.000000f, 0.000000f, 0.165810f, 0.526162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086343f, 0.166394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028875f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068792f, 0.750632f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.180576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.798640f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401325f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.563541f, 0.035134f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393109f, 0.035360f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.514780f, 0.056751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.286324f, 0.066048f, 0.000000f, 0.022966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.397320f, 0.167136f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018733f, 0.017081f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228689f, 0.212401f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027812f, 0.000000f, 0.230123f, 0.251307f, 0.000000f, 0.015952f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018366f, 0.015349f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.089768f, 0.272262f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.165947f, 0.450195f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021828f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064329f, 0.394519f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021491f, 0.519661f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.420154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579846f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.561993f, 0.042727f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395280f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.507366f, 0.060806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.388432f, 0.043397f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.017057f, 0.019075f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.399856f, 0.181694f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.283918f, 0.098400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018320f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.261768f, 0.263599f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210680f, 0.218119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019283f, 0.018776f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156143f, 0.407378f, 0.000000f, 0.018410f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.081168f, 0.298842f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043712f, 0.524648f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025861f, 0.405779f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027775f, 0.567781f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.404444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797266f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.164849f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.736579f, 0.098573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.139627f, 0.082102f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.529383f, 0.220315f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020496f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031087f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029563f, 0.000000f, 0.069934f, 0.077745f, 0.000000f, 0.000000f, 0.000000f, 0.019031f, 0.000000f, 0.000000f, 0.369058f, 0.383087f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072848f, 0.128566f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206674f, 0.591912f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028891f, 0.164765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.054845f, 0.751498f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.186782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.813218f}, +}; + +// For each output (8x4) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x4[32][48] = { +{0.800445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.801084f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198916f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.802438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.800166f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199834f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.808142f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191858f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.801414f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198586f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.798600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.800453f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199547f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415774f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.584226f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.407361f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.592639f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411487f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.588513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.416734f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.583266f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590206f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409782f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.419797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.580203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.588149f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.411851f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.591287f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.408713f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.587561f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.412439f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.589820f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410180f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.585460f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.414540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.590541f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.409459f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.587115f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.412885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.584462f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415538f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.799529f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195628f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.804372f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195562f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.804438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.194079f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.805921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.205775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.794225f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197129f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.802871f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193175f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.806825f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185493f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.814507f}, +}; + +// For each output (2x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x5[10][48] = { +{0.314987f, 0.280141f, 0.203583f, 0.129696f, 0.071593f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.085378f, 0.141565f, 0.188187f, 0.272403f, 0.312467f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.255395f, 0.217105f, 0.170584f, 0.106646f, 0.072684f, 0.000000f, 0.000000f, 0.000000f, 0.072766f, 0.046537f, 0.029920f, 0.000000f, 0.028363f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069530f, 0.105913f, 0.164044f, 0.215260f, 0.255339f, 0.000000f, 0.000000f, 0.000000f, 0.025591f, 0.000000f, 0.036814f, 0.050349f, 0.077160f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152274f, 0.142699f, 0.102993f, 0.080565f, 0.018558f, 0.000000f, 0.000000f, 0.000000f, 0.157267f, 0.135460f, 0.099077f, 0.089287f, 0.021820f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026396f, 0.087011f, 0.099835f, 0.143472f, 0.149274f, 0.000000f, 0.000000f, 0.000000f, 0.019143f, 0.078700f, 0.099557f, 0.143621f, 0.152993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071546f, 0.054560f, 0.034641f, 0.000000f, 0.026492f, 0.000000f, 0.000000f, 0.000000f, 0.253751f, 0.217970f, 0.167740f, 0.101477f, 0.071823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031122f, 0.000000f, 0.038539f, 0.044578f, 0.068079f, 0.000000f, 0.000000f, 0.000000f, 0.074011f, 0.104132f, 0.176778f, 0.213248f, 0.249513f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.309516f, 0.271823f, 0.202932f, 0.138334f, 0.077394f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073235f, 0.136322f, 0.204986f, 0.270837f, 0.314620f}, +}; + +// For each output (3x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x5[15][48] = { +{0.506870f, 0.329427f, 0.163702f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.029175f, 0.167327f, 0.319880f, 0.321166f, 0.162451f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158719f, 0.334975f, 0.506306f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410647f, 0.270965f, 0.135943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.101890f, 0.048392f, 0.032162f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022675f, 0.131363f, 0.257700f, 0.263834f, 0.126043f, 0.021278f, 0.000000f, 0.000000f, 0.000000f, 0.022613f, 0.064121f, 0.066389f, 0.023985f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.131149f, 0.266568f, 0.407438f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.041342f, 0.046648f, 0.106854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.259144f, 0.176197f, 0.070648f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.256402f, 0.170550f, 0.067060f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085864f, 0.160352f, 0.153663f, 0.093488f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.093065f, 0.165400f, 0.162870f, 0.085298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069632f, 0.177258f, 0.252242f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066495f, 0.178932f, 0.255440f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109165f, 0.056989f, 0.043673f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.396795f, 0.263538f, 0.129840f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022525f, 0.061369f, 0.062101f, 0.020335f, 0.000000f, 0.000000f, 0.000000f, 0.022912f, 0.129308f, 0.258462f, 0.259250f, 0.129291f, 0.034446f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.042198f, 0.051815f, 0.111374f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.136459f, 0.257176f, 0.400979f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.509094f, 0.334982f, 0.155925f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.175231f, 0.321060f, 0.327712f, 0.175997f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154955f, 0.336566f, 0.508479f}, +}; + +// For each output (4x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x5[20][48] = { +{0.669318f, 0.330682f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.147967f, 0.437694f, 0.317636f, 0.064825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031879f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.060625f, 0.318845f, 0.433756f, 0.158597f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324316f, 0.675684f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.585012f, 0.264010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150977f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.134170f, 0.326735f, 0.247128f, 0.055953f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060565f, 0.080612f, 0.050606f, 0.022675f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065736f, 0.255091f, 0.336456f, 0.141260f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020320f, 0.056879f, 0.083295f, 0.040963f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247404f, 0.561749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037270f, 0.153576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.313615f, 0.178768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.317328f, 0.167805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.056200f, 0.226923f, 0.169203f, 0.032339f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060880f, 0.227803f, 0.168145f, 0.036277f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022230f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020809f, 0.161103f, 0.242215f, 0.080276f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037660f, 0.170123f, 0.226083f, 0.061733f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170517f, 0.314573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.183677f, 0.312560f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018674f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150066f, 0.037627f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.563093f, 0.249214f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017288f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047237f, 0.083719f, 0.064159f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.141594f, 0.343865f, 0.254176f, 0.047961f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060771f, 0.083714f, 0.056548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055519f, 0.260450f, 0.341460f, 0.141538f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033365f, 0.158801f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243363f, 0.564471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027870f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.650693f, 0.321437f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.154390f, 0.455517f, 0.321763f, 0.068330f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030540f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.067841f, 0.315774f, 0.431982f, 0.153863f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029780f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.315631f, 0.654589f}, +}; + +// For each output (5x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x5[25][48] = { +{0.728974f, 0.241827f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029199f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.326790f, 0.583809f, 0.061650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027751f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.474659f, 0.471971f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027161f, 0.026208f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.064479f, 0.600103f, 0.335418f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245795f, 0.727343f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026862f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.577450f, 0.212083f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.146821f, 0.063646f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278532f, 0.501669f, 0.039082f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.051617f, 0.129101f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401558f, 0.402789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088129f, 0.087552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039177f, 0.470310f, 0.275467f, 0.000000f, 0.000000f, 0.000000f, 0.020182f, 0.000000f, 0.000000f, 0.131064f, 0.041994f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021806f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.201719f, 0.586252f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071189f, 0.140839f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390859f, 0.113288f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.395284f, 0.100569f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.180479f, 0.291419f, 0.034269f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179460f, 0.288259f, 0.026114f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.232294f, 0.235881f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249972f, 0.265992f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.015860f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020495f, 0.297441f, 0.200057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.300629f, 0.181378f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.094856f, 0.384959f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.114338f, 0.382484f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023363f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.142672f, 0.067752f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579242f, 0.210334f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050987f, 0.132705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.278585f, 0.484125f, 0.053597f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026554f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.092842f, 0.065201f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.385798f, 0.387342f, 0.000000f, 0.000000f, 0.021183f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021080f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020712f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.044924f, 0.106062f, 0.061499f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047893f, 0.466019f, 0.252890f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020637f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.058939f, 0.143896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202796f, 0.573732f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033403f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.730809f, 0.235788f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032140f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.330176f, 0.584667f, 0.053018f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026110f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.492274f, 0.481616f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065854f, 0.592001f, 0.342145f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037025f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.240768f, 0.722207f}, +}; + +// For each output (6x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x5[30][48] = { +{0.858351f, 0.111195f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030454f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.561719f, 0.406108f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032173f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.234049f, 0.720564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045387f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.699282f, 0.247085f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.053633f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.389024f, 0.574352f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.092315f, 0.907685f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.700837f, 0.094616f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.181782f, 0.022766f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478824f, 0.322377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.106995f, 0.067586f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.020740f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019187f, 0.000000f, 0.211821f, 0.554939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.076920f, 0.116393f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.528826f, 0.215423f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.129030f, 0.084167f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021007f, 0.021548f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.285851f, 0.511729f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.045516f, 0.156904f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061737f, 0.729570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.185199f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023495f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.426048f, 0.065346f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.437353f, 0.050722f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.015946f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.269275f, 0.220699f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.271762f, 0.222318f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107929f, 0.387609f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.097175f, 0.384787f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022500f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018661f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.393619f, 0.098786f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.415799f, 0.073135f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.219562f, 0.256847f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.228262f, 0.295329f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.020203f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066094f, 0.437807f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023625f, 0.426898f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025372f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179453f, 0.029939f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.702329f, 0.088278f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024531f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.109211f, 0.062119f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.483375f, 0.320765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017885f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077080f, 0.134573f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212908f, 0.535331f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022223f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.119888f, 0.115275f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.556098f, 0.208739f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022346f, 0.116179f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.324515f, 0.536960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.039522f, 0.193447f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.040639f, 0.726391f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033823f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.857552f, 0.108625f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029799f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.542169f, 0.403976f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052699f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.223511f, 0.723790f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052693f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.702269f, 0.245038f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.402547f, 0.597453f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031996f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086881f, 0.881123f}, +}; + +// For each output (7x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x5[35][48] = { +{0.964445f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.853417f, 0.094561f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052022f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.657134f, 0.277797f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020663f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023601f, 0.000000f, 0.000000f, 0.020806f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.380325f, 0.419839f, 0.000000f, 0.023060f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032462f, 0.000000f, 0.000000f, 0.025415f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022865f, 0.000000f, 0.028258f, 0.000000f, 0.023082f, 0.020352f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024341f, 0.000000f, 0.000000f}, +{0.000000f, 0.031003f, 0.000000f, 0.000000f, 0.218422f, 0.657212f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024308f, 0.033400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035654f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070868f, 0.871307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.964400f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.771715f, 0.027473f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200812f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.681017f, 0.087709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.170219f, 0.037187f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023867f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019162f, 0.000000f, 0.019267f, 0.000000f, 0.521425f, 0.210553f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.107845f, 0.064833f, 0.000000f, 0.000000f, 0.000000f, 0.023456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016876f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016582f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.374490f, 0.378533f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.037317f, 0.000000f, 0.070870f, 0.081690f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019460f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020149f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017492f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198514f, 0.553647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069444f, 0.178395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077267f, 0.707241f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.191176f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024316f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.777498f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.197118f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025384f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.457893f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.477045f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024793f, 0.020109f, 0.000000f, 0.020160f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.453272f, 0.036882f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.449988f, 0.037704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.390518f, 0.119870f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.380701f, 0.108911f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017868f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.216278f, 0.228953f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.240939f, 0.263209f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016253f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029917f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.096934f, 0.340899f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.088970f, 0.426562f, 0.000000f, 0.000000f, 0.016718f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021872f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073754f, 0.459232f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.422925f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022217f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.019775f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.473981f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020534f, 0.461485f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024225f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.200471f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.772740f, 0.026789f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025642f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.165170f, 0.033854f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.660678f, 0.089428f, 0.000000f, 0.000000f, 0.000000f, 0.025229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.016453f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.117847f, 0.083344f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.528281f, 0.230342f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023732f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.043833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077971f, 0.049154f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.382849f, 0.385195f, 0.000000f, 0.022790f, 0.000000f, 0.000000f, 0.020308f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017900f}, +{0.000000f, 0.000000f, 0.018444f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.017477f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.086693f, 0.093631f, 0.000000f, 0.032653f, 0.000000f, 0.000000f, 0.019144f, 0.000000f, 0.199637f, 0.532319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020247f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.035464f, 0.208022f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065940f, 0.670327f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.209616f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.790384f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036613f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.963387f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.046570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.849248f, 0.104183f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020833f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.049999f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.649521f, 0.279647f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030284f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025099f, 0.000000f, 0.000000f, 0.017993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028953f, 0.000000f, 0.027848f, 0.031988f, 0.000000f, 0.000000f, 0.000000f, 0.022049f, 0.000000f, 0.000000f, 0.397216f, 0.418570f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026723f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.038960f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243424f, 0.690894f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.050705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.071869f, 0.877426f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036401f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.963599f}, +}; + +// For each output (8x5) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x5[40][48] = { +{0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.966306f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033694f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.966298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033702f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966295f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.793476f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803849f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196151f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803624f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196376f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797993f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202007f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.776552f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.195983f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027465f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.793721f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.206279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.806466f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.193534f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.797656f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.202344f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.476380f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.496730f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026890f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.490205f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.485068f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024727f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.498077f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.476651f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025272f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.474340f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.480228f, 0.045432f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478505f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.521495f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478679f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.483579f, 0.000000f, 0.000000f, 0.037742f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.521456f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.478544f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.507379f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.492621f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.204896f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.795104f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.196765f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.803235f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.199650f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.800350f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.203568f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.796432f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.179104f, 0.025788f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.795108f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.198542f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.801458f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.212749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.787251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.210279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.789721f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966296f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033709f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966291f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033700f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966300f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033711f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966289f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033705f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966295f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033692f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966308f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033717f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966283f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.966269f}, +}; + +// For each output (2x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_2x6[12][48] = { +{0.316864f, 0.281020f, 0.203578f, 0.128737f, 0.069800f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.084099f, 0.140260f, 0.188810f, 0.272909f, 0.313922f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.309774f, 0.274434f, 0.201401f, 0.144203f, 0.070188f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065514f, 0.142636f, 0.201399f, 0.276345f, 0.314107f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.317592f, 0.277500f, 0.192959f, 0.141457f, 0.070491f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073241f, 0.142588f, 0.198561f, 0.278233f, 0.307377f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320020f, 0.275328f, 0.193983f, 0.143663f, 0.067007f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.069519f, 0.132193f, 0.205168f, 0.279209f, 0.313912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.314759f, 0.279613f, 0.202284f, 0.130432f, 0.072912f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.077965f, 0.136688f, 0.207007f, 0.271208f, 0.307132f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.311744f, 0.272206f, 0.202758f, 0.136022f, 0.077269f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.072611f, 0.134437f, 0.204577f, 0.271631f, 0.316744f}, +}; + +// For each output (3x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_3x6[18][48] = { +{0.509323f, 0.329513f, 0.161164f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.025207f, 0.165943f, 0.323432f, 0.324818f, 0.160600f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.157414f, 0.335022f, 0.507564f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.511584f, 0.329744f, 0.158672f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031983f, 0.159222f, 0.310218f, 0.312506f, 0.158287f, 0.027785f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156210f, 0.333357f, 0.510434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.515123f, 0.331176f, 0.153701f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026619f, 0.155693f, 0.312956f, 0.312469f, 0.159059f, 0.033204f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.156669f, 0.330733f, 0.512598f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.503816f, 0.332794f, 0.163390f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024597f, 0.154193f, 0.318347f, 0.305757f, 0.159499f, 0.037605f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.158978f, 0.332267f, 0.508755f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.512301f, 0.329905f, 0.157794f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.034639f, 0.152702f, 0.307204f, 0.309309f, 0.167621f, 0.028524f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152238f, 0.331031f, 0.516731f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.511179f, 0.335760f, 0.153061f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.173463f, 0.322489f, 0.329811f, 0.174238f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152159f, 0.337011f, 0.510830f}, +}; + +// For each output (4x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_4x6[24][48] = { +{0.671100f, 0.328900f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.148979f, 0.456693f, 0.330185f, 0.064143f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.058158f, 0.330805f, 0.451065f, 0.159972f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322150f, 0.677850f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.677593f, 0.322407f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.167723f, 0.446276f, 0.319975f, 0.066025f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.073990f, 0.323047f, 0.441943f, 0.161020f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.326071f, 0.673929f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.679042f, 0.320958f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.152853f, 0.450375f, 0.323919f, 0.072853f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.061203f, 0.320863f, 0.451270f, 0.166664f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.319746f, 0.680254f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.676510f, 0.323490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.162624f, 0.457726f, 0.332137f, 0.047514f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063329f, 0.328068f, 0.444798f, 0.163805f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.320574f, 0.679426f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.678066f, 0.321934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.166497f, 0.448536f, 0.320669f, 0.064298f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.065578f, 0.323791f, 0.452649f, 0.157982f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322175f, 0.677825f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.671500f, 0.328500f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.150795f, 0.460955f, 0.323971f, 0.064280f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.066061f, 0.327767f, 0.449877f, 0.156295f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.322687f, 0.677313f}, +}; + +// For each output (5x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_5x6[30][48] = { +{0.754364f, 0.245636f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.335285f, 0.602164f, 0.062551f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.500479f, 0.499521f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.057582f, 0.607199f, 0.335218f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249634f, 0.750366f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.757244f, 0.242756f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.346204f, 0.598435f, 0.055362f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.501490f, 0.498510f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.060219f, 0.591314f, 0.348467f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.244713f, 0.755287f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.752634f, 0.247366f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.342331f, 0.595920f, 0.061748f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.496285f, 0.503715f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.055875f, 0.601113f, 0.343013f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245684f, 0.754316f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.754642f, 0.245358f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.341881f, 0.605457f, 0.052662f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.506471f, 0.493529f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.052276f, 0.594038f, 0.353686f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243659f, 0.756341f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.752998f, 0.247002f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.343161f, 0.587149f, 0.069691f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.497737f, 0.502263f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068745f, 0.600800f, 0.330455f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.249755f, 0.750245f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.760155f, 0.239845f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.341132f, 0.607027f, 0.051841f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.505602f, 0.494398f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.063784f, 0.594541f, 0.341675f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.246784f, 0.753216f}, +}; + +// For each output (6x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_6x6[36][48] = { +{0.891095f, 0.108905f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.581832f, 0.418168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.242153f, 0.757847f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.741976f, 0.258024f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.403606f, 0.596394f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.087517f, 0.912483f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.889771f, 0.110229f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.562123f, 0.416930f, 0.000000f, 0.020947f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.239798f, 0.760202f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.745430f, 0.254570f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.386117f, 0.613883f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.079820f, 0.920180f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.881826f, 0.118174f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.573611f, 0.426389f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.253276f, 0.746724f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743647f, 0.256353f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401870f, 0.598130f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084584f, 0.915416f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.886496f, 0.113504f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.579329f, 0.420671f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.247079f, 0.752921f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.738480f, 0.261520f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.387849f, 0.612151f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084296f, 0.915704f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.887045f, 0.112955f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.566292f, 0.413182f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020526f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.245603f, 0.754397f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743664f, 0.256336f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.400389f, 0.599611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.085951f, 0.914049f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.893377f, 0.106623f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023576f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.559870f, 0.416555f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.230693f, 0.769307f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.743815f, 0.256185f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.401590f, 0.598410f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.084902f, 0.915098f}, +}; + +// For each output (7x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_7x6[42][48] = { +{1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.898749f, 0.101251f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.666832f, 0.285944f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024418f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.022807f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.408751f, 0.452880f, 0.000000f, 0.022279f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020101f, 0.000000f, 0.026406f, 0.000000f, 0.021392f, 0.021638f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026554f, 0.000000f, 0.000000f}, +{0.000000f, 0.030824f, 0.000000f, 0.000000f, 0.224222f, 0.683355f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.025094f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.036505f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.074156f, 0.925844f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.898226f, 0.101774f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026159f, 0.000000f, 0.029283f, 0.000000f, 0.659538f, 0.261285f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023736f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.465472f, 0.460934f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.047490f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026105f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.265328f, 0.711299f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023373f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.070803f, 0.929197f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.898251f, 0.101749f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.031089f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.687826f, 0.256342f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024743f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.032537f, 0.022458f, 0.000000f, 0.000000f, 0.021138f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.410489f, 0.430095f, 0.000000f, 0.017967f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018781f, 0.000000f, 0.000000f, 0.026567f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019968f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.246938f, 0.753062f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075058f, 0.924942f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.901297f, 0.098703f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.030068f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.643429f, 0.251859f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020470f, 0.024955f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.029220f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018980f, 0.000000f, 0.000000f, 0.026284f, 0.019861f, 0.000000f, 0.028010f, 0.000000f, 0.000000f, 0.000000f, 0.445381f, 0.461483f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.238827f, 0.737044f, 0.000000f, 0.024129f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.068567f, 0.931433f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.900397f, 0.099603f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.026865f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.019850f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.657029f, 0.271313f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.024943f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.033697f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.020611f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026549f, 0.000000f, 0.440563f, 0.453523f, 0.000000f, 0.025057f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.030966f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.255214f, 0.713821f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075429f, 0.924571f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.889492f, 0.110508f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.021347f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.683322f, 0.295331f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.027842f, 0.000000f, 0.000000f, 0.031873f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.028367f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.023945f, 0.000000f, 0.000000f, 0.417773f, 0.443199f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.026004f, 0.000000f, 0.000000f, 0.024319f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.243621f, 0.706056f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.075547f, 0.924453f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f}, +}; + +// For each output (8x6) sample, the weight of each input (8x6) sample. +static const float g_weight_downsample_8x6_to_8x6[48][48] = { +{1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f, 0.000000f}, +{0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 1.000000f}, +}; + +downsample_matrix g_downsample_matrices_8x6[] = +{ + { 2, 2, (const float*)g_weight_downsample_8x6_to_2x2 }, + { 3, 2, (const float*)g_weight_downsample_8x6_to_3x2 }, + { 4, 2, (const float*)g_weight_downsample_8x6_to_4x2 }, + { 5, 2, (const float*)g_weight_downsample_8x6_to_5x2 }, + { 6, 2, (const float*)g_weight_downsample_8x6_to_6x2 }, + { 7, 2, (const float*)g_weight_downsample_8x6_to_7x2 }, + { 8, 2, (const float*)g_weight_downsample_8x6_to_8x2 }, + { 2, 3, (const float*)g_weight_downsample_8x6_to_2x3 }, + { 3, 3, (const float*)g_weight_downsample_8x6_to_3x3 }, + { 4, 3, (const float*)g_weight_downsample_8x6_to_4x3 }, + { 5, 3, (const float*)g_weight_downsample_8x6_to_5x3 }, + { 6, 3, (const float*)g_weight_downsample_8x6_to_6x3 }, + { 7, 3, (const float*)g_weight_downsample_8x6_to_7x3 }, + { 8, 3, (const float*)g_weight_downsample_8x6_to_8x3 }, + { 2, 4, (const float*)g_weight_downsample_8x6_to_2x4 }, + { 3, 4, (const float*)g_weight_downsample_8x6_to_3x4 }, + { 4, 4, (const float*)g_weight_downsample_8x6_to_4x4 }, + { 5, 4, (const float*)g_weight_downsample_8x6_to_5x4 }, + { 6, 4, (const float*)g_weight_downsample_8x6_to_6x4 }, + { 7, 4, (const float*)g_weight_downsample_8x6_to_7x4 }, + { 8, 4, (const float*)g_weight_downsample_8x6_to_8x4 }, + { 2, 5, (const float*)g_weight_downsample_8x6_to_2x5 }, + { 3, 5, (const float*)g_weight_downsample_8x6_to_3x5 }, + { 4, 5, (const float*)g_weight_downsample_8x6_to_4x5 }, + { 5, 5, (const float*)g_weight_downsample_8x6_to_5x5 }, + { 6, 5, (const float*)g_weight_downsample_8x6_to_6x5 }, + { 7, 5, (const float*)g_weight_downsample_8x6_to_7x5 }, + { 8, 5, (const float*)g_weight_downsample_8x6_to_8x5 }, + { 2, 6, (const float*)g_weight_downsample_8x6_to_2x6 }, + { 3, 6, (const float*)g_weight_downsample_8x6_to_3x6 }, + { 4, 6, (const float*)g_weight_downsample_8x6_to_4x6 }, + { 5, 6, (const float*)g_weight_downsample_8x6_to_5x6 }, + { 6, 6, (const float*)g_weight_downsample_8x6_to_6x6 }, + { 7, 6, (const float*)g_weight_downsample_8x6_to_7x6 }, + { 8, 6, (const float*)g_weight_downsample_8x6_to_8x6 } +}; + //-------------------------------------------------------------------------------------------------------------------------- const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height) { - // TODO: Use hash or map lookup. + // TODO: Use hash or map lookup, or calc the index directly for (const auto& m : g_downsample_matrices_6x6) if ((m.m_grid_width == grid_width) && (m.m_grid_height == grid_height)) return m.m_p; @@ -4766,6 +5668,121 @@ const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height return nullptr; } +const float* get_8x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height) +{ + // TODO: Use hash or map lookup, or calc the index directly + for (const auto& m : g_downsample_matrices_8x6) + if ((m.m_grid_width == grid_width) && (m.m_grid_height == grid_height)) + return m.m_p; + + assert(0); + return nullptr; +} + +//-------------------------------------------------------------------------------------------------------------------------- + +void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) +{ + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + astc_helpers::weighted_sample samples[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + clear_obj(samples); + + astc_helpers::compute_upsample_weights(block_width, block_height, grid_width, grid_height, samples); + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + upsample_matrix.resize_rows_cols(num_block_samples, num_grid_samples); + + basisu::vector weights(num_grid_samples); + + // compute which source sample(s) contribute to it. + for (uint32_t d = 0; d < num_block_samples; d++) + { + const astc_helpers::weighted_sample& ws = samples[d]; + + weights.set_all(0.0f); + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + float w = ws.m_weights[y][x] * (1.0f / 16.0f); + if (!w) + continue; + + assert((ws.m_src_x + x) < grid_width); + assert((ws.m_src_y + y) < grid_height); + + assert(weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] == 0.0f); + weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] = w; + } // x + } // y + + for (uint32_t i = 0; i < num_grid_samples; i++) + upsample_matrix.at_row_col(d, i) = weights[i]; + + } // d +} + +//-------------------------------------------------------------------------------------------------------------------------- +// compute At - used for gradient descent + +void compute_upsample_matrix_transposed(basisu::vector& unweighted_downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) +{ + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, block_width, block_height, grid_width, grid_height); + + // downsample matrix At (without any scaling): num_grid_samples (rows), num_block_samples (cols) + unweighted_downsample_matrix.resize(num_grid_samples * num_block_samples); + unweighted_downsample_matrix.set_all(0.0f); + + for (uint32_t j = 0; j < num_grid_samples; ++j) + for (uint32_t i = 0; i < num_block_samples; ++i) + unweighted_downsample_matrix[j * num_block_samples + i] = upsample_matrix.at_row_col(i, j); +} + +//-------------------------------------------------------------------------------------------------------------------------- +// Computes downsample matrices - simpler alternative to SLSQP + +//-------------------------------------------------------------------------------------------------------------------------- +// pDst_vec[] - size must be >= num_grid_samples +// vector used for gradient descent + +void compute_diag_AtA_vector(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height, const vector2D &upsample_matrix, float* pDst_vec) +{ + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + memset(pDst_vec, 0, sizeof(float) * num_grid_samples); + + for (uint32_t r = 0; r < num_block_samples; ++r) + { + for (uint32_t c = 0; c < num_grid_samples; ++c) + { + const float arc = upsample_matrix.at_row_col(r, c); + + pDst_vec[c] += arc * arc; + } + } +} + +//-------------------------------------------------------------------------------------------------------------------------- + void downsample_weight_grid( const float* pMatrix_weights, uint32_t bx, uint32_t by, // source/from dimension (block size) @@ -4803,7 +5820,7 @@ void downsample_ise_weights( assert((block_w <= MAX_ASTC_HDR_BLOCK_W) && (block_h <= MAX_ASTC_HDR_BLOCK_H)); assert((grid_w >= 2) && (grid_w <= MAX_ASTC_HDR_BLOCK_W)); assert((grid_h >= 2) && (grid_h <= MAX_ASTC_HDR_BLOCK_H)); - + assert(dequant_weight_ise_range >= astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE); assert(dequant_weight_ise_range <= astc_helpers::LAST_VALID_WEIGHT_ISE_RANGE); @@ -4902,7 +5919,7 @@ static bool refine_endpoints_mode11( { for (uint32_t i = 0; i < num_block_pixels; i++) def_pixel_block_ofs[i] = (uint8_t)i; - + pPixel_block_ofs = def_pixel_block_ofs; } @@ -4928,7 +5945,7 @@ static bool refine_endpoints_mode11( trial_blk_raw_weights[i] = upsampled_weights[pPixel_block_ofs[i]]; trial_blk_raw_weightsf[i] = (float)trial_blk_raw_weights[i] * (1.0f / 64.0f); } - + vec3F l_q16, h_q16; if (opt_mode == cOrdinaryLeastSquares) { @@ -4959,7 +5976,7 @@ static bool refine_endpoints_mode11( { float mid = (0.0f - l) / (h - l); mid = clamp(mid, .01f, .99f); - + float lw = LOW_EMPHASIS_WEIGHT, mw = MIDDLE_EMPHASIS_WEIGHT, hw = HIGH_EMPHASIS_WEIGHT; if (opt_mode == cWeightedLeastSquaresHeavy) lw = LOW_EMPHASIS_WEIGHT_HEAVY, mw = MIDDLE_EMPHASIS_WEIGHT_HEAVY, hw = HIGH_EMPHASIS_WEIGHT_HEAVY; @@ -5019,7 +6036,7 @@ static bool refine_endpoints_mode11( } uint8_t trial_endpoints[NUM_MODE11_ENDPOINTS]; - + uint32_t submode_used; bool pack_succeeded = pack_mode11(l_q16, h_q16, endpoint_ise_range, trial_endpoints, coptions, direct_only, first_submode, last_submode, false, submode_used); @@ -5046,7 +6063,7 @@ static bool refine_endpoints_mode11( const float R_WEIGHT = coptions.m_r_err_scale, G_WEIGHT = coptions.m_g_err_scale; double cur_error = 0, trial_error = 0; - + for (uint32_t p = 0; p < num_pixels; p++) { const half_float* pDesired_half = &pBlock_pixels_half[p][0]; @@ -5173,7 +6190,7 @@ static bool refine_endpoints_mode7( vec3F block_mean_color_q16(calc_mean(num_pixels, pBlock_pixels_q16)); vec3F new_high_color_q16(block_mean_color_q16); - + const float one_over_num_pixels = 1.0f / (float)num_pixels; for (uint32_t i = 0; i < num_pixels; i++) @@ -5185,7 +6202,7 @@ static bool refine_endpoints_mode7( new_high_color_q16[1] += k; new_high_color_q16[2] += k; } - + // Given a set of selectors and a high color, try to compute a better S. float t = 0.0f; @@ -5197,7 +6214,7 @@ static bool refine_endpoints_mode7( } t *= one_over_num_pixels; - + if (fabs(t) < .0000125f) return false; @@ -5223,7 +6240,7 @@ static bool refine_endpoints_mode7( if (!decode_mode7_to_qlog12(trial_endpoints, trial_e, nullptr, endpoint_ise_range)) return false; - + // -- for (uint32_t i = 0; i < 3; i++) @@ -5354,3 +6371,4 @@ bool refine_endpoints( } } // namespace basisu + diff --git a/encoder/basisu_astc_hdr_common.h b/encoder/basisu_astc_hdr_common.h index 18f424f..e019992 100644 --- a/encoder/basisu_astc_hdr_common.h +++ b/encoder/basisu_astc_hdr_common.h @@ -12,13 +12,13 @@ namespace basisu const uint32_t MODE11_TOTAL_SUBMODES = 8; // plus an extra hidden submode, directly encoded, for direct, so really 9 (see tables 99/100 of the ASTC spec) const uint32_t MODE7_TOTAL_SUBMODES = 6; - + // [ise_range][0] = # levels // [ise_range][1...] = lerp value [0,64] // in ASTC order // Supported ISE weight ranges: 0 to 11, 12 total const uint32_t MIN_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_2_LEVELS; // ISE 0=2 levels - const uint32_t MAX_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_32_LEVELS; // ISE 11=16 levels + const uint32_t MAX_SUPPORTED_ISE_WEIGHT_INDEX = astc_helpers::BISE_32_LEVELS; // ISE 11=32 levels const uint32_t MIN_SUPPORTED_WEIGHT_LEVELS = 2; const uint32_t MAX_SUPPORTED_WEIGHT_LEVELS = 32; @@ -29,13 +29,17 @@ namespace basisu const float LDR_TO_HDR_NITS = 100.0f; + extern vec4F g_astc_ls_weights_ise[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; + extern uint8_t g_map_astc_to_linear_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][astc_index] -> linear index + extern uint8_t g_map_linear_to_astc_order[MAX_SUPPORTED_ISE_WEIGHT_INDEX + 1][MAX_SUPPORTED_WEIGHT_LEVELS]; // [ise_range][linear_index] -> astc_index + struct astc_hdr_codec_base_options { float m_r_err_scale, m_g_err_scale; float m_q_log_bias; - + bool m_ultra_quant; - + // If true, the ASTC HDR compressor is allowed to more aggressively vary weight indices for slightly higher compression in non-fastest mode. This will hurt BC6H quality, however. bool m_allow_uber_mode; @@ -45,7 +49,7 @@ namespace basisu bool m_take_first_non_clamping_mode7_submode; bool m_disable_weight_plane_optimization; - + astc_hdr_codec_base_options() { init(); } void init(); @@ -173,7 +177,7 @@ namespace basisu basist::half_float* pDecoded_half, vec3F* pDecoded_float, uint32_t n, uint32_t ise_weight_range, uint32_t ise_endpoint_range); - + // Fast high precision piecewise linear approximation of log2(bias+x). // Half may be zero, positive or denormal. No NaN/Inf/negative. BASISU_FORCE_INLINE double q(basist::half_float x, float log_bias) @@ -183,7 +187,7 @@ namespace basisu fi.f = fast_half_to_float_pos_not_inf_or_nan(x); assert(fi.f >= 0.0f); - + fi.f += log_bias; return (double)fi.u; // approx log2f(fi.f), need to return double for the precision @@ -196,7 +200,7 @@ namespace basisu fi.f = fast_half_to_float_pos_not_inf_or_nan(x); assert(fi.f >= 0.0f); - + fi.f += log_bias; return fi.u; @@ -298,8 +302,8 @@ namespace basisu uint32_t ise_endpoint_range, bool uber_mode, bool constrain_ise_weight_selectors, - int32_t first_submode, int32_t last_submode, bool ignore_clamping, - opt_mode_t opt_mode, + int32_t first_submode, int32_t last_submode, bool ignore_clamping, + opt_mode_t opt_mode, const encode_astc_block_stats *pBlock_stats = nullptr); double encode_astc_hdr_block_downsampled_mode_11( @@ -325,7 +329,7 @@ namespace basisu uint32_t ise_endpoint_range, bool uber_mode, bool constrain_ise_weight_selectors, - int32_t first_submode, int32_t last_submode, + int32_t first_submode, int32_t last_submode, bool ignore_clamping); double encode_astc_hdr_block_mode_7( @@ -337,8 +341,8 @@ namespace basisu uint8_t* blk_endpoints, //[4] uint8_t* blk_weights, // [num_pixels] const astc_hdr_codec_base_options& coptions, - uint32_t ise_endpoint_range, - int first_submode = 0, int last_submode = MAX_MODE7_SUBMODE_INDEX, + uint32_t ise_endpoint_range, + int first_submode = 0, int last_submode = MAX_MODE7_SUBMODE_INDEX, const encode_astc_block_stats *pBlock_stats = nullptr); //-------------------------------------------------------------------------------------------------------------------------- @@ -373,17 +377,23 @@ namespace basisu bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, int val_q[2][3], int& max_clamp_mag, bool early_out_if_clamped = false, int max_clamp_mag_accept_thresh = 0); bool pack_astc_mode11_submode(uint32_t submode, uint8_t* pEndpoints, const vec3F& low_q16, const vec3F& high_q16, int& max_clamp_mag, bool early_out_if_clamped = false, int max_clamp_mag_accept_thresh = 0); void pack_astc_mode11_direct(uint8_t* pEndpoints, vec3F l_q16, vec3F h_q16); - + bool pack_mode11(mode11_log_desc& desc, uint8_t* pEndpoints); void unpack_mode11(const uint8_t* pEndpoints, mode11_log_desc& desc); void decode_cem_11_config(const uint8_t* pEndpoints, int& submode_index, int& maj_index); void decode_cem_7_config(const uint8_t* pEndpoints, int& submode_index, int& maj_index); - + void dequantize_astc_weights(uint32_t n, const uint8_t* pSrc_ise_vals, uint32_t from_ise_range, uint8_t* pDst_raw_weights); const float* get_6x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height); - + const float* get_8x6_downsample_matrix(uint32_t grid_width, uint32_t grid_height); + + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + void compute_upsample_matrix_transposed(basisu::vector& unweighted_downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + void compute_diag_AtA_vector(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height, const vector2D& upsample_matrix, float* pDst_vec); + void downsample_weight_grid( const float* pMatrix_weights, uint32_t bx, uint32_t by, // source/from dimension (block size) @@ -413,10 +423,11 @@ namespace basisu uint32_t num_pixels, const basist::half_float pBlock_pixels_half[][3], const vec4F pBlock_pixels_q16[], const uint8_t* pPixel_block_ofs, // maps this subset's pixels to block offsets astc_hdr_codec_base_options& coptions, opt_mode_t opt_mode); - + extern bool g_astc_hdr_enc_initialized; // This MUST be called before encoding any blocks. void astc_hdr_enc_init(); } // namespace basisu + diff --git a/encoder/basisu_backend.cpp b/encoder/basisu_backend.cpp index 8096e25..adc791e 100644 --- a/encoder/basisu_backend.cpp +++ b/encoder/basisu_backend.cpp @@ -1,5 +1,5 @@ // basisu_backend.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ namespace basisu m_pFront_end = pFront_end; m_params = params; m_slices = slice_descs; - + debug_printf("basisu_backend::Init: Slices: %u, ETC1S: %u, EndpointRDOQualityThresh: %f, SelectorRDOQualityThresh: %f\n", m_slices.size(), params.m_etc1s, @@ -196,7 +196,7 @@ namespace basisu m_endpoint_remap_table_old_to_new = reorderer.get_remap_table(); } - // For endpoints, old_to_new[] may not be bijective! + // For endpoints, old_to_new[] may not be bijective! // Some "old" entries may be unused and don't get remapped into the "new" array. m_old_endpoint_was_used.clear(); @@ -220,13 +220,13 @@ namespace basisu } // slice_index debug_printf("basisu_backend::reoptimize_and_sort_endpoints_codebook: First old entry index: %u\n", first_old_entry_index); - + m_new_endpoint_was_used.clear(); m_new_endpoint_was_used.resize(r.get_total_endpoint_clusters()); m_endpoint_remap_table_new_to_old.clear(); m_endpoint_remap_table_new_to_old.resize(r.get_total_endpoint_clusters()); - + // Set unused entries in the new array to point to the first used entry in the old array. m_endpoint_remap_table_new_to_old.set_all(first_old_entry_index); @@ -235,7 +235,7 @@ namespace basisu if (m_old_endpoint_was_used[old_index]) { const uint32_t new_index = m_endpoint_remap_table_old_to_new[old_index]; - + m_new_endpoint_was_used[new_index] = true; m_endpoint_remap_table_new_to_old[new_index] = old_index; @@ -612,7 +612,7 @@ namespace basisu sort_selector_codebook(); check_for_valid_cr_blocks(); - + debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); } @@ -666,14 +666,14 @@ namespace basisu if (m_params.m_debug_images) { image gi_unpacked; - gi.unpack(gi_unpacked); + gi.unpack(gi_unpacked, false); char buf[256]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(buf, sizeof(buf), "basisu_backend_slice_%u.png", slice_index); #else snprintf(buf, sizeof(buf), "basisu_backend_slice_%u.png", slice_index); -#endif +#endif save_png(buf, gi_unpacked); } @@ -682,7 +682,7 @@ namespace basisu //uint32_t g_color_delta_hist[255 * 3 + 1]; //uint32_t g_color_delta_bad_hist[255 * 3 + 1]; - + // TODO: Split this into multiple methods. bool basisu_backend::encode_image() { @@ -718,7 +718,7 @@ namespace basisu const int COLOR_DELTA_THRESH = 8; const int SEL_DIFF_THRESHOLD = 11; - + for (uint32_t slice_index = 0; slice_index < m_slices.size(); slice_index++) { //const int prev_frame_slice_index = is_video ? find_video_frame(slice_index, -1) : -1; @@ -764,7 +764,7 @@ namespace basisu } // block_x } // block_y - + for (uint32_t block_y = 0; block_y < num_blocks_y; block_y++) { for (uint32_t block_x = 0; block_x < num_blocks_x; block_x++) @@ -842,7 +842,7 @@ namespace basisu const uint32_t cur_inten5 = etc_blk.get_inten_table(0); const etc1_endpoint_palette_entry& cur_endpoints = m_endpoint_palette[m.m_endpoint_index]; - + if (cur_err) { const float endpoint_remap_thresh = maximum(1.0f, m_params.m_endpoint_rdo_quality_thresh); @@ -858,7 +858,7 @@ namespace basisu int best_trial_idx = 0; etc_block trial_etc_blk(etc_blk); - + const int search_dist = minimum(iabs(endpoint_delta) - 1, MAX_ENDPOINT_SEARCH_DIST); for (int d = -search_dist; d < search_dist; d++) { @@ -876,7 +876,7 @@ namespace basisu continue; const etc1_endpoint_palette_entry& p = m_endpoint_palette[m_endpoint_remap_table_new_to_old[trial_idx]]; - + if (m_params.m_compression_level <= 1) { if (p.m_inten5 > cur_inten5) @@ -886,7 +886,7 @@ namespace basisu int delta_g = iabs(cur_endpoints.m_color5.g - p.m_color5.g); int delta_b = iabs(cur_endpoints.m_color5.b - p.m_color5.b); int color_delta = delta_r + delta_g + delta_b; - + if (color_delta > COLOR_DELTA_THRESH) continue; } @@ -924,7 +924,7 @@ namespace basisu const int64_t initial_best_trial_err = INT64_MAX; int64_t best_trial_err = initial_best_trial_err; int best_trial_idx = 0; - + const int search_dist = minimum(iabs(endpoint_delta) - 1, MAX_ENDPOINT_SEARCH_DIST); for (int d = -search_dist; d < search_dist; d++) { @@ -942,7 +942,7 @@ namespace basisu continue; const etc1_endpoint_palette_entry& p = m_endpoint_palette[m_endpoint_remap_table_new_to_old[trial_idx]]; - + if (m_params.m_compression_level <= 1) { if (p.m_inten5 > cur_inten5) @@ -952,7 +952,7 @@ namespace basisu int delta_g = iabs(cur_endpoints.m_color5.g - p.m_color5.g); int delta_b = iabs(cur_endpoints.m_color5.b - p.m_color5.b); int color_delta = delta_r + delta_g + delta_b; - + if (color_delta > COLOR_DELTA_THRESH) continue; } @@ -992,7 +992,7 @@ namespace basisu } #endif // BASISU_SUPPORT_SSE } // if (!g_cpu_supports_sse41) - + } // if (cur_err) } // if ((m_params.m_endpoint_rdo_quality_thresh > 1.0f) && (iabs(endpoint_delta) > 1) && (!block_endpoints_are_referenced(block_x, block_y))) @@ -1011,7 +1011,7 @@ namespace basisu if ((!is_video) || (m.m_endpoint_predictor != basist::CR_ENDPOINT_PRED_INDEX)) { int new_selector_index = m_selector_remap_table_old_to_new[m.m_selector_index]; - + const float selector_remap_thresh = maximum(1.0f, m_params.m_selector_rdo_quality_thresh); //2.5f; int selector_history_buf_index = -1; @@ -1060,7 +1060,7 @@ namespace basisu for (uint32_t p = 0; p < 16; p++) cur_err += color_distance(false, src_pixels.get_ptr()[p], block_colors[pCur_selectors[p]], false); } - + const uint64_t limit_err = (uint64_t)ceilf(cur_err * selector_remap_thresh); // Even if cur_err==limit_err, we still want to scan the history buffer because there may be equivalent entries that are cheaper to code. @@ -1091,7 +1091,7 @@ namespace basisu if (sel_diff >= SEL_DIFF_THRESHOLD) continue; } - + const uint64_t thresh_err = minimum(limit_err, best_trial_err); uint64_t trial_err = 0; @@ -1266,7 +1266,7 @@ namespace basisu //{ // printf("%u, %u, %f\n", g_color_delta_bad_hist[i], g_color_delta_hist[i], g_color_delta_hist[i] ? g_color_delta_bad_hist[i] / (float)g_color_delta_hist[i] : 0); //} - + double total_prep_time = tm.get_elapsed_secs(); debug_printf("basisu_backend::encode_image: Total prep time: %3.2f\n", total_prep_time); @@ -1521,7 +1521,7 @@ namespace basisu if (old_endpoint_was_used[old_endpoint_index]) { const uint32_t new_endpoint_index = m_endpoint_remap_table_old_to_new[old_endpoint_index]; - + new_endpoint_was_used[new_endpoint_index] = true; endpoint_remap_table_new_to_old[new_endpoint_index] = old_endpoint_index; @@ -1660,7 +1660,7 @@ namespace basisu bool basisu_backend::encode_selector_palette() { const basisu_frontend& r = *m_pFront_end; - + histogram delta_selector_pal_histogram(256); for (uint32_t q = 0; q < r.get_total_selector_clusters(); q++) diff --git a/encoder/basisu_comp.cpp b/encoder/basisu_comp.cpp index 9a52f54..617ff30 100644 --- a/encoder/basisu_comp.cpp +++ b/encoder/basisu_comp.cpp @@ -1,5 +1,5 @@ // basisu_comp.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ #include "basisu_miniz.h" #include "basisu_opencl.h" +#include "basisu_astc_ldr_encode.h" #include "../transcoder/basisu_astc_hdr_core.h" @@ -50,11 +51,207 @@ using namespace buminiz; namespace basisu { + static float uastc_ldr_4x4_lambda_from_quality(float q) + { + q = clamp(q, 0.0f, 1.0f); + + if (q >= 1.0f) + return 0.0f; + + const float lambda_max = 20.0f; + return lambda_max * pow(1.0f - q, 1.3f); + } + + static float uastc_hdr_6x6_lambda_from_quality(float q) + { + // Ideally we would know if it's an upconverted LDR/SDR input, or HDR, then that controls the maximum useful lambda. + q = clamp(q, 0.0f, 1.0f); + + if (q >= 1.0f) + return 0.0f; + + const float lambda_max = 50000.0f; + return lambda_max * pow(1.0f - q, 1.5f); + } + + bool basis_compressor_params::set_format_mode_and_effort(basist::basis_tex_format mode, int effort, bool set_defaults) + { + fmt_debug_printf("set_format_mode_and_effort: mode: {}, effort: {}, set_defaults: {}\n", basist::basis_get_tex_format_name(mode), effort, set_defaults); + + set_format_mode(mode); + + if (effort > 0) + effort = clamp(effort, 0, 10); + + const float feffort = (effort >= 0) ? clamp((float)effort / 10.0f, 0.0f, 1.0f) : 0.0f; + + if (mode == basist::basis_tex_format::cETC1S) + { + if (effort >= 0) + m_etc1s_compression_level = (int)std::round(lerp(0, (float)BASISU_MAX_ETC1S_COMPRESSION_LEVEL, feffort)); + else if (set_defaults) + m_etc1s_compression_level = BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL; + + fmt_debug_printf("Low-level ETC1S compression (effort) level (0-6): {}\n", m_etc1s_compression_level); + } + else if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + if (effort >= 0) + m_pack_uastc_ldr_4x4_flags = (int)std::round(lerp((float)cPackUASTCLevelFastest, (float)cPackUASTCLevelVerySlow, feffort)); + else if (set_defaults) + m_pack_uastc_ldr_4x4_flags = cPackUASTCLevelDefault; + + fmt_debug_printf("Low-level UASTC LDR 4x4 pack (effort) level (0-4): {}\n", m_pack_uastc_ldr_4x4_flags); + } + else if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) + { + // Set UASTC HDR 4x4 effort level (there is no quality to set - it doesn't support RDO yet). + if (effort >= 0) + m_uastc_hdr_4x4_options.set_quality_level((int)std::round(lerp((float)uastc_hdr_4x4_codec_options::cMinLevel, (float)uastc_hdr_4x4_codec_options::cMaxLevel, feffort))); + else if (set_defaults) + m_uastc_hdr_4x4_options.set_quality_level(uastc_hdr_4x4_codec_options::cDefaultLevel); + + fmt_debug_printf("Low-level UASTC HDR 4x4 quality (actually effort) level (0-4): {}\n", m_uastc_hdr_4x4_options.m_level); + } + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) + { + // Set ASTC HDR 6x6/UASTC HDR 6x6 effort level + if (effort >= 0) + m_astc_hdr_6x6_options.set_user_level(effort); + else if (set_defaults) + m_astc_hdr_6x6_options.set_user_level(astc_6x6_hdr::ASTC_HDR_6X6_DEF_USER_COMP_LEVEL); + + fmt_debug_printf("Low-level UASTC HDR 6x6 master comp (effort) level (0-4): {}, highest comp (effort) level (0-4): {}, num reuse XY deltas: {}, extra patterns flag: {}, brute force partition matching: {}\n", + m_astc_hdr_6x6_options.m_master_comp_level, + m_astc_hdr_6x6_options.m_highest_comp_level, + m_astc_hdr_6x6_options.m_num_reuse_xy_deltas, + m_astc_hdr_6x6_options.m_extra_patterns_flag, + m_astc_hdr_6x6_options.m_brute_force_partition_matching); + } + else if ((mode >= basist::basis_tex_format::cXUASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cASTC_LDR_12x12)) + { + if (effort >= 0) + m_xuastc_ldr_effort_level = effort; + else if (set_defaults) + m_xuastc_ldr_effort_level = astc_ldr::EFFORT_LEVEL_DEF; + + fmt_debug_printf("Low-level XUASTC LDR effort level (0-10): {}\n", m_xuastc_ldr_effort_level); + } + else + { + assert(0); + return false; + } + + return true; + } + + bool basis_compressor_params::set_format_mode_and_quality_effort(basist::basis_tex_format mode, int quality, int effort, bool set_defaults) + { + fmt_debug_printf("set_format_mode_and_quality_effort: mode: {}, quality: {}, effort: {}, set_defaults: {}\n", basist::basis_get_tex_format_name(mode), quality, effort, set_defaults); + + if (!set_format_mode_and_effort(mode, effort, set_defaults)) + return false; + + if (quality > 0) + quality = clamp(quality, 0, 100); + + const float fquality = (quality >= 0) ? clamp((float)quality / 100.0f, 0.0f, 1.0f) : 0.0f; + + if (mode == basist::basis_tex_format::cETC1S) + { + // ETC1S: Map quality and effort to ETC1S quality and effort levels + if (quality >= 0) + m_quality_level = (int)std::round(lerp(0, 255.0f, fquality)); + else if (set_defaults) + m_quality_level = -1; + + fmt_debug_printf("Low-level ETC1S quality level (0-255): {}\n", m_quality_level); + } + else if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + // UASTC LDR 4x4: Map quality to RDO lambda scalar, effort to UASTC LDR 4x4 packing level + if ((quality >= 0) && (quality < 100)) + { + // Enable RDO postprocessing + m_rdo_uastc_ldr_4x4 = true; + + // Attempt to derive a reasonable lambda from quality + m_rdo_uastc_ldr_4x4_quality_scalar = uastc_ldr_4x4_lambda_from_quality(fquality); + } + else if (set_defaults) + { + m_rdo_uastc_ldr_4x4 = false; + + m_rdo_uastc_ldr_4x4_quality_scalar = 1.0f; // the default is 1.0, but the RDO flag isn't enabled + } + + fmt_debug_printf("Low-level UASTC LDR 4x4 RDO flag: {}, lambda setting (0=no extra distortion, higher=more distortion): {}\n", m_rdo_uastc_ldr_4x4, m_rdo_uastc_ldr_4x4_quality_scalar); + } + else if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) + { + // UASTC HDR 4x4: Nothing to do for quality, it doesn't support RDO + if ((quality != -1) && (quality < 100)) + { + fmt_printf("WARNING: UASTC HDR 4x4 codec doesn't have a 'quality' parameter (it doesn't currently support RDO)\n"); + } + } + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) + { + // Set lambda (rate-distortion tradeoff) + if (quality >= 0) + m_astc_hdr_6x6_options.m_lambda = uastc_hdr_6x6_lambda_from_quality(fquality); + else if (set_defaults) + m_astc_hdr_6x6_options.m_lambda = 0.0f; + + fmt_debug_printf("Low-level UASTC HDR 6x6 lambda setting (0=no extra distortion, higher=more distortion): {}\n", m_astc_hdr_6x6_options.m_lambda); + } + else if ((mode >= basist::basis_tex_format::cASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cASTC_LDR_12x12)) + { + // ASTC LDR 4x4-12x12: Nothing to do for quality, it doesn't support RDO + if ((quality != -1) && (quality < 100)) + { + fmt_printf("WARNING: ASTC LDR 4x4-12x12 codec doesn't have a 'quality' parameter (it doesn't currently support RDO)\n"); + } + } + else if ((mode >= basist::basis_tex_format::cXUASTC_LDR_4x4) && (mode <= basist::basis_tex_format::cXUASTC_LDR_12x12)) + { + // XUASTC LDR 4x4-12x12 + if ((quality >= 0) && (quality < 100)) + { + // Enable DCT + lossy supercompression + m_quality_level = quality; + m_xuastc_ldr_use_dct = true; + m_xuastc_ldr_use_lossy_supercompression = true; + } + else if (set_defaults) + { + m_quality_level = -1; + m_xuastc_ldr_use_dct = false; + m_xuastc_ldr_use_lossy_supercompression = false; + } + + fmt_debug_printf("Low-level XUASTC quality level (0-100): {}, Use DCT: {}, Use lossy supercompression: {}\n", m_quality_level, m_xuastc_ldr_use_dct, m_xuastc_ldr_use_lossy_supercompression); + } + else + { + assert(0); + return false; + } + + return true; + } + basis_compressor::basis_compressor() : m_pOpenCL_context(nullptr), m_fmt_mode(basist::basis_tex_format::cETC1S), + m_fmt_mode_block_width(4), + m_fmt_mode_block_height(4), + m_total_slice_orig_texels(0), m_basis_file_size(0), m_basis_bits_per_texel(0.0f), + m_ktx2_file_size(0), + m_ktx2_bits_per_texel(0.0f), m_total_blocks(0), m_hdr_image_scale(1.0f), m_ldr_to_hdr_upconversion_nit_multiplier(1.0f), @@ -63,7 +260,7 @@ namespace basisu m_opencl_failed(false) { debug_printf("basis_compressor::basis_compressor\n"); - + assert(g_library_initialized); } @@ -195,7 +392,7 @@ namespace basisu bool basis_compressor::init(const basis_compressor_params ¶ms) { debug_printf("basis_compressor::init\n"); - + if (!g_library_initialized) { error_printf("basis_compressor::init: basisu_encoder_init() MUST be called before using any encoder functionality!\n"); @@ -207,7 +404,7 @@ namespace basisu error_printf("basis_compressor::init: A non-null job_pool pointer must be specified\n"); return false; } - + m_params = params; if ((m_params.m_compute_stats) && (!m_params.m_validate_output_data)) @@ -217,17 +414,34 @@ namespace basisu m_ldr_to_hdr_upconversion_nit_multiplier = 1.0f; m_upconverted_any_ldr_images = false; + m_total_slice_orig_texels = 0; + m_basis_file_size = 0; + m_basis_bits_per_texel = 0.0f; + m_ktx2_file_size = 0; + m_ktx2_bits_per_texel = 0.0f; + check_for_hdr_inputs(); + if (m_params.m_hdr) + { + if ((m_params.m_debug) && (m_params.m_ktx2_and_basis_srgb_transfer_function) && (m_params.m_ktx2_and_basis_srgb_transfer_function.was_changed())) + { + debug_printf("Warning: m_ktx2_and_basis_srgb_transfer_function being forced to false in HDR mode (we always write linear KTX2/.basis files in HDR mode)\n"); + } + + // Always slam m_ktx2_and_basis_srgb_transfer_function on HDR inputs. We always write linear to KTX2 and .basis for HDR outputs. + m_params.m_ktx2_and_basis_srgb_transfer_function = false; + } + if (m_params.m_debug) { - debug_printf("basis_compressor::init:\n"); + debug_printf("\nbasis_compressor::init:\n"); #define PRINT_BOOL_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_INT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_UINT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); #define PRINT_FLOAT_VALUE(v) fmt_debug_printf("{}: {} {}\n", BASISU_STRINGIZE2(v), static_cast(m_params.v), m_params.v.was_changed()); - + fmt_debug_printf("Source LDR images: {}, HDR images: {}, filenames: {}, alpha filenames: {}, LDR mipmap images: {}, HDR mipmap images: {}\n", (uint64_t)m_params.m_source_images.size(), (uint64_t)m_params.m_source_images_hdr.size(), (uint64_t)m_params.m_source_filenames.size(), (uint64_t)m_params.m_source_alpha_filenames.size(), @@ -263,9 +477,9 @@ namespace basisu fmt_debug_printf("m_hdr_mode: cASTC_HDR_6X6\n"); break; } - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: { - fmt_debug_printf("m_hdr_mode: cASTC_HDR_6X6_INTERMEDIATE\n"); + fmt_debug_printf("m_hdr_mode: cUASTC_HDR_6X6_INTERMEDIATE\n"); break; } default: @@ -274,12 +488,13 @@ namespace basisu } PRINT_BOOL_VALUE(m_uastc); + PRINT_INT_VALUE(m_xuastc_or_astc_ldr_basis_tex_format); PRINT_BOOL_VALUE(m_use_opencl); PRINT_BOOL_VALUE(m_y_flip); PRINT_BOOL_VALUE(m_debug); PRINT_BOOL_VALUE(m_validate_etc1s); PRINT_BOOL_VALUE(m_debug_images); - PRINT_INT_VALUE(m_compression_level); + PRINT_INT_VALUE(m_etc1s_compression_level); PRINT_BOOL_VALUE(m_perceptual); PRINT_BOOL_VALUE(m_no_endpoint_rdo); PRINT_BOOL_VALUE(m_no_selector_rdo); @@ -296,10 +511,10 @@ namespace basisu PRINT_BOOL_VALUE(m_renormalize); PRINT_BOOL_VALUE(m_multithreading); PRINT_BOOL_VALUE(m_disable_hierarchical_endpoint_codebooks); - + PRINT_FLOAT_VALUE(m_endpoint_rdo_thresh); PRINT_FLOAT_VALUE(m_selector_rdo_thresh); - + PRINT_BOOL_VALUE(m_mip_gen); PRINT_BOOL_VALUE(m_mip_renormalize); PRINT_BOOL_VALUE(m_mip_wrapping); @@ -312,14 +527,14 @@ namespace basisu debug_printf("m_max_endpoint_clusters: %u\n", m_params.m_etc1s_max_endpoint_clusters); debug_printf("m_max_selector_clusters: %u\n", m_params.m_etc1s_max_selector_clusters); - debug_printf("m_etc1s_quality_level: %i\n", m_params.m_etc1s_quality_level); + debug_printf("m_quality_level: %i\n", m_params.m_quality_level); debug_printf("UASTC HDR 4x4 quality level: %u\n", m_params.m_uastc_hdr_4x4_options.m_level); debug_printf("m_tex_type: %u\n", m_params.m_tex_type); debug_printf("m_userdata0: 0x%X, m_userdata1: 0x%X\n", m_params.m_userdata0, m_params.m_userdata1); debug_printf("m_us_per_frame: %i (%f fps)\n", m_params.m_us_per_frame, m_params.m_us_per_frame ? 1.0f / (m_params.m_us_per_frame / 1000000.0f) : 0); debug_printf("m_pack_uastc_ldr_4x4_flags: 0x%X\n", m_params.m_pack_uastc_ldr_4x4_flags); - + PRINT_BOOL_VALUE(m_rdo_uastc_ldr_4x4); PRINT_FLOAT_VALUE(m_rdo_uastc_ldr_4x4_quality_scalar); PRINT_INT_VALUE(m_rdo_uastc_ldr_4x4_dict_size); @@ -333,7 +548,7 @@ namespace basisu PRINT_INT_VALUE(m_resample_width); PRINT_INT_VALUE(m_resample_height); PRINT_FLOAT_VALUE(m_resample_factor); - + debug_printf("Has global codebooks: %u\n", m_params.m_pGlobal_codebooks ? 1 : 0); if (m_params.m_pGlobal_codebooks) { @@ -344,7 +559,7 @@ namespace basisu debug_printf("KTX2 UASTC supercompression: %u\n", m_params.m_ktx2_uastc_supercompression); debug_printf("KTX2 Zstd supercompression level: %i\n", (int)m_params.m_ktx2_zstd_supercompression_level); - debug_printf("KTX2 sRGB transfer func: %u\n", (int)m_params.m_ktx2_srgb_transfer_func); + debug_printf("KTX2/basis sRGB transfer function: %u\n", (int)m_params.m_ktx2_and_basis_srgb_transfer_function); debug_printf("Total KTX2 key values: %u\n", m_params.m_ktx2_key_values.size()); for (uint32_t i = 0; i < m_params.m_ktx2_key_values.size(); i++) { @@ -353,21 +568,45 @@ namespace basisu } PRINT_BOOL_VALUE(m_validate_output_data); + PRINT_UINT_VALUE(m_transcode_flags); PRINT_BOOL_VALUE(m_ldr_hdr_upconversion_srgb_to_linear); PRINT_FLOAT_VALUE(m_ldr_hdr_upconversion_nit_multiplier); debug_printf("Allow UASTC HDR 4x4 uber mode: %u\n", m_params.m_uastc_hdr_4x4_options.m_allow_uber_mode); debug_printf("UASTC HDR 4x4 ultra quant: %u\n", m_params.m_uastc_hdr_4x4_options.m_ultra_quant); PRINT_BOOL_VALUE(m_hdr_favor_astc); + + PRINT_INT_VALUE(m_xuastc_ldr_effort_level); + PRINT_BOOL_VALUE(m_xuastc_ldr_blurring); + PRINT_BOOL_VALUE(m_xuastc_ldr_use_dct); + PRINT_BOOL_VALUE(m_xuastc_ldr_use_lossy_supercompression); + PRINT_BOOL_VALUE(m_xuastc_ldr_force_disable_subsets); + PRINT_BOOL_VALUE(m_xuastc_ldr_force_disable_rgb_dual_plane); + PRINT_INT_VALUE(m_xuastc_ldr_syntax); + debug_printf("XUASTC LDR channel weights: "); + for (uint32_t i = 0; i < 4; i++) + fmt_debug_printf("{} ", m_params.m_xuastc_ldr_channel_weights[i]); + debug_printf("\n"); + + PRINT_FLOAT_VALUE(m_ls_min_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_edge_psnr); + PRINT_FLOAT_VALUE(m_ls_min_alpha_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_alpha_psnr); + PRINT_FLOAT_VALUE(m_ls_thresh_edge_alpha_psnr); + #undef PRINT_BOOL_VALUE #undef PRINT_INT_VALUE #undef PRINT_UINT_VALUE #undef PRINT_FLOAT_VALUE + + fmt_printf("m_format_mode: {}\n", (uint32_t)m_params.get_format_mode()); + fmt_printf("\n"); } if (!sanity_check_input_params()) return false; - + if ((m_params.m_use_opencl) && opencl_is_available() && !m_pOpenCL_context && !m_opencl_failed) { m_pOpenCL_context = opencl_create_context(); @@ -378,14 +617,17 @@ namespace basisu return true; } - void basis_compressor::pick_format_mode() + bool basis_compressor::pick_format_mode() { - // Unfortunately due to the legacy of this code and backwards compat this is more complex than I would like. + // Unfortunately due to the legacy of this code and backwards API compatibility this is more complex than I would like. m_fmt_mode = basist::basis_tex_format::cETC1S; + m_fmt_mode_block_width = 4; + m_fmt_mode_block_height = 4; if (m_params.m_hdr) { assert(m_params.m_uastc); + assert(m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1); switch (m_params.m_hdr_mode) { @@ -394,9 +636,13 @@ namespace basisu break; case hdr_modes::cASTC_HDR_6X6: m_fmt_mode = basist::basis_tex_format::cASTC_HDR_6x6; + m_fmt_mode_block_width = 6; + m_fmt_mode_block_height = 6; break; - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - m_fmt_mode = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: + m_fmt_mode = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; + m_fmt_mode_block_width = 6; + m_fmt_mode_block_height = 6; break; default: assert(0); @@ -405,7 +651,30 @@ namespace basisu } else if (m_params.m_uastc) { - m_fmt_mode = basist::basis_tex_format::cUASTC4x4; + if (m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1) + { + // UASTC LDR 4x4 + m_fmt_mode = basist::basis_tex_format::cUASTC_LDR_4x4; + } + else + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + m_fmt_mode = static_cast(static_cast(m_params.m_xuastc_or_astc_ldr_basis_tex_format)); + + if (!basis_tex_format_is_xuastc_ldr(m_fmt_mode) && !basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + assert(0); + error_printf("basis_compressor::pick_format_mode: m_xuastc_or_astc_ldr_basis_tex_format is invalid\n"); + return false; + } + + basist::get_basis_tex_format_block_size(m_fmt_mode, m_fmt_mode_block_width, m_fmt_mode_block_height); + } + } + else + { + // ETC1S + assert(m_params.m_xuastc_or_astc_ldr_basis_tex_format == -1); } if (m_params.m_debug) @@ -415,8 +684,8 @@ namespace basisu case basist::basis_tex_format::cETC1S: fmt_debug_printf("Format Mode: cETC1S\n"); break; - case basist::basis_tex_format::cUASTC4x4: - fmt_debug_printf("Format Mode: cUASTC4x4\n"); + case basist::basis_tex_format::cUASTC_LDR_4x4: + fmt_debug_printf("Format Mode: cUASTC_LDR_4x4\n"); break; case basist::basis_tex_format::cUASTC_HDR_4x4: fmt_debug_printf("Format Mode: cUASTC_HDR_4x4\n"); @@ -424,16 +693,56 @@ namespace basisu case basist::basis_tex_format::cASTC_HDR_6x6: fmt_debug_printf("Format Mode: cASTC_HDR_6x6\n"); break; - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: - fmt_debug_printf("Format Mode: cASTC_HDR_6x6_INTERMEDIATE\n"); + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: + fmt_debug_printf("Format Mode: cUASTC_HDR_6x6_INTERMEDIATE\n"); break; + + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + fmt_debug_printf("Format Mode: cXUASTC_LDR_{}x{}\n", m_fmt_mode_block_width, m_fmt_mode_block_height); + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + fmt_debug_printf("Format Mode: cASTC_LDR_{}x{}\n", m_fmt_mode_block_width, m_fmt_mode_block_height); + break; + } + default: assert(0); break; } } - } + return true; + } + basis_compressor::error_code basis_compressor::process() { debug_printf("basis_compressor::process\n"); @@ -441,8 +750,9 @@ namespace basisu if (!read_dds_source_images()) return cECFailedReadingSourceImages; - // Note: After here m_params.m_hdr, m_params.m_uastc and m_fmt_mode cannot be changed. - pick_format_mode(); + // Note: After here m_params.m_hdr, m_params.m_uastc and m_fmt_mode, m_fmt_mode_block_width/height cannot be changed. + if (!pick_format_mode()) + return cECFailedInvalidParameters; if (!read_source_images()) return cECFailedReadingSourceImages; @@ -459,6 +769,7 @@ namespace basisu } } + // Some modes/codecs require extracting source blocks up front. if (!extract_source_blocks()) return cECFailedFrontEnd; @@ -468,21 +779,21 @@ namespace basisu { // UASTC 4x4 HDR if (m_params.m_status_output) - printf("Mode: UASTC 4x4 HDR Level %u\n", m_params.m_uastc_hdr_4x4_options.m_level); + printf("Mode: UASTC 4x4 HDR Effort Level (0-4): %u\n", m_params.m_uastc_hdr_4x4_options.m_level); error_code ec = encode_slices_to_uastc_4x4_hdr(); if (ec != cECSuccess) return ec; } - else + else { - assert((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE)); + assert((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE)); // ASTC 6x6 HDR if (m_params.m_status_output) { - fmt_printf("Mode: ASTC 6x6 HDR {}, Base Level: {}, Highest Level: {}, Lambda: {}, REC 2020: {}\n", - (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE) ? "Intermediate" : "", + fmt_printf("Mode: ASTC 6x6 HDR {}, Base Effort Level (0-4): {}, Highest Effort Level (0-4): {}, Lambda: {}, REC 2020: {}\n", + (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE) ? "Intermediate" : "", m_params.m_astc_hdr_6x6_options.m_master_comp_level, m_params.m_astc_hdr_6x6_options.m_highest_comp_level, m_params.m_astc_hdr_6x6_options.m_lambda, m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut); } @@ -494,11 +805,54 @@ namespace basisu } else if (m_params.m_uastc) { - // UASTC 4x4 LDR - if (m_params.m_status_output) - printf("Mode: UASTC LDR 4x4 Level %u\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask); + error_code ec = cECFailedEncodeUASTC; + + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + if (m_params.m_status_output) + { + uint32_t block_width = 0, block_height = 0; + basist::get_basis_tex_format_block_size(m_fmt_mode, block_width, block_height); + + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode)) + { + fmt_printf("Mode: XUASTC LDR {}x{}, Effort Level (0-10): {}, Disable Subsets: {}, Disable RGB Dual Plane: {}\nWeight grid DCT: {}, DCT quality level (1-100): {}, Lossy supercompression: {}, sRGB8 ASTC decode profile: {}, Syntax: {}, Channel weights: {} {} {} {}\n", + block_width, block_height, (int)m_params.m_xuastc_ldr_effort_level, (bool)m_params.m_xuastc_ldr_force_disable_subsets, (bool)m_params.m_xuastc_ldr_force_disable_rgb_dual_plane, + (bool)m_params.m_xuastc_ldr_use_dct, + (bool)m_params.m_xuastc_ldr_use_dct ? m_params.m_quality_level : 0, + (bool)m_params.m_xuastc_ldr_use_lossy_supercompression, + (bool)m_params.m_ktx2_and_basis_srgb_transfer_function, + (int)m_params.m_xuastc_ldr_syntax, + m_params.m_xuastc_ldr_channel_weights[0], m_params.m_xuastc_ldr_channel_weights[1], m_params.m_xuastc_ldr_channel_weights[2], m_params.m_xuastc_ldr_channel_weights[3]); + } + else + { + fmt_printf("Mode: ASTC LDR {}x{}, Effort Level (0-10): {}, Disable Subsets: {}, Disable RGB Dual Plane: {}, sRGB8 ASTC decode profile: {}, Syntax: {}, Channel weights: {} {} {} {}\n", + block_width, block_height, + (int)m_params.m_xuastc_ldr_effort_level, (bool)m_params.m_xuastc_ldr_force_disable_subsets, (bool)m_params.m_xuastc_ldr_force_disable_rgb_dual_plane, + (bool)m_params.m_ktx2_and_basis_srgb_transfer_function, + (int)m_params.m_xuastc_ldr_syntax, + m_params.m_xuastc_ldr_channel_weights[0], m_params.m_xuastc_ldr_channel_weights[1], m_params.m_xuastc_ldr_channel_weights[2], m_params.m_xuastc_ldr_channel_weights[3]); + } + } + + ec = encode_slices_to_xuastc_or_astc_ldr(); + } + else + { + // UASTC LDR 4x4 + if (m_params.m_status_output) + { + if (m_params.m_rdo_uastc_ldr_4x4) + fmt_printf("Mode: UASTC LDR 4x4 Effort Level (0-4): {}, using RDO lambda: {}\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask, m_params.m_rdo_uastc_ldr_4x4_quality_scalar); + else + printf("Mode: UASTC LDR 4x4 Effort Level (0-4): %u\n", m_params.m_pack_uastc_ldr_4x4_flags & cPackUASTCLevelMask); + } + + ec = encode_slices_to_uastc_4x4_ldr(); + } - error_code ec = encode_slices_to_uastc_4x4_ldr(); if (ec != cECSuccess) return ec; } @@ -506,8 +860,8 @@ namespace basisu { // ETC1S if (m_params.m_status_output) - printf("Mode: ETC1S Quality %i, Level %i\n", m_params.m_etc1s_quality_level, (int)m_params.m_compression_level); - + printf("Mode: ETC1S Quality (1-255): %i, Comp Level (Effort, 0-6): %i\n", m_params.m_quality_level, (int)m_params.m_etc1s_compression_level); + if (!process_frontend()) return cECFailedFrontEnd; @@ -546,14 +900,14 @@ namespace basisu if (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cASTC_HDR_6x6; - else if (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE) - m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + else if (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE) + m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; else { assert(0); return cECFailedEncodeUASTC; } - + m_uastc_backend_output.m_etc1s = false; m_uastc_backend_output.m_srgb = false; m_uastc_backend_output.m_slice_desc = m_slice_descs; @@ -561,13 +915,13 @@ namespace basisu m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); astc_6x6_hdr::astc_hdr_6x6_global_config global_cfg(m_params.m_astc_hdr_6x6_options); - + global_cfg.m_image_stats = m_params.m_compute_stats; global_cfg.m_debug_images = m_params.m_debug_images; global_cfg.m_output_images = m_params.m_debug_images; global_cfg.m_debug_output = m_params.m_debug; global_cfg.m_status_output = m_params.m_status_output || m_params.m_debug; - + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& dst_tex = m_uastc_slice_textures[slice_index]; @@ -578,7 +932,7 @@ namespace basisu const imagef& source_image = m_slice_images_hdr[slice_index]; assert(source_image.get_width() && source_image.get_height()); - + uint8_vec intermediate_tex_data, astc_tex_data; global_cfg.m_debug_image_prefix = m_params.m_astc_hdr_6x6_options.m_debug_image_prefix; @@ -586,7 +940,7 @@ namespace basisu global_cfg.m_output_image_prefix = m_params.m_astc_hdr_6x6_options.m_output_image_prefix; global_cfg.m_output_image_prefix += fmt_string("slice_{}_", slice_index); - + if (m_params.m_debug) fmt_debug_printf("----------------------------------------------------------------------------\n"); @@ -613,7 +967,7 @@ namespace basisu } else { - assert(m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE); + assert(m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE); dst_buf.resize(intermediate_tex_data.size_in_bytes()); memcpy(&dst_buf[0], intermediate_tex_data.get_ptr(), intermediate_tex_data.size_in_bytes()); @@ -648,7 +1002,7 @@ namespace basisu m_params.m_uastc_hdr_4x4_options.m_r_err_scale = 1.0f; m_params.m_uastc_hdr_4x4_options.m_g_err_scale = 1.0f; } - + const float DEFAULT_BC6H_ERROR_WEIGHT = .65f;// .85f; const float LOWEST_BC6H_ERROR_WEIGHT = .1f; m_params.m_uastc_hdr_4x4_options.m_bc6h_err_weight = m_params.m_hdr_favor_astc ? LOWEST_BC6H_ERROR_WEIGHT : DEFAULT_BC6H_ERROR_WEIGHT; @@ -657,7 +1011,7 @@ namespace basisu any_failures.store(false); astc_hdr_4x4_block_stats enc_stats; - + struct uastc_blk_desc { uint32_t m_solid_flag; @@ -681,7 +1035,7 @@ namespace basisu return false; } - + bool operator== (const uastc_blk_desc& desc) const { if (this == &desc) @@ -709,6 +1063,10 @@ namespace basisu std::map unique_block_descs; std::mutex unique_block_desc_mutex; + std::mutex status_output_mutex; + uint32_t total_blocks_processed = 0; + float last_percentage_printed = 0; + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& tex = m_uastc_slice_textures[slice_index]; @@ -719,18 +1077,16 @@ namespace basisu const uint32_t num_blocks_y = tex.get_blocks_y(); const uint32_t total_blocks = tex.get_total_blocks(); const imagef& source_image = m_slice_images_hdr[slice_index]; - - std::atomic total_blocks_processed; - total_blocks_processed.store(0); - + const uint32_t N = 256; for (uint32_t block_index_iter = 0; block_index_iter < total_blocks; block_index_iter += N) { const uint32_t first_index = block_index_iter; const uint32_t last_index = minimum(total_blocks, block_index_iter + N); - - m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, - &tex, &total_blocks_processed, &any_failures, &enc_stats, &unique_block_descs, &unique_block_desc_mutex] + + m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, + &tex, &any_failures, &enc_stats, &unique_block_descs, &unique_block_desc_mutex, + &status_output_mutex, &total_blocks_processed, &last_percentage_printed] { BASISU_NOTE_UNUSED(num_blocks_y); @@ -750,7 +1106,7 @@ namespace basisu source_image.extract_block_clamped(&block_pixels[0], block_x * 4, block_y * 4, 4, 4); basist::astc_blk& dest_block = *(basist::astc_blk*)tex.get_block_ptr(block_x, block_y); - + float rgb_pixels[16 * 3]; basist::half_float rgb_pixels_half[16 * 3]; for (uint32_t i = 0; i < 16; i++) @@ -764,7 +1120,7 @@ namespace basisu rgb_pixels[i * 3 + 2] = block_pixels[i][2]; rgb_pixels_half[i * 3 + 2] = float_to_half_non_neg_no_nan_inf(block_pixels[i][2]); } - + bool status = astc_hdr_4x4_enc_block(&rgb_pixels[0], rgb_pixels_half, m_params.m_uastc_hdr_4x4_options, all_results); if (!status) { @@ -774,10 +1130,10 @@ namespace basisu double best_err = 1e+30f; int best_result_index = -1; - + const double bc6h_err_weight = m_params.m_uastc_hdr_4x4_options.m_bc6h_err_weight; const double astc_err_weight = (1.0f - bc6h_err_weight); - + for (uint32_t i = 0; i < all_results.size(); i++) { basist::half_float unpacked_bc6h_block[4 * 4 * 3]; @@ -795,9 +1151,9 @@ namespace basisu } const astc_hdr_4x4_pack_results& best_results = all_results[best_result_index]; - + astc_hdr_4x4_pack_results_to_block(dest_block, best_results); - + // Verify that this block is valid UASTC HDR and we can successfully transcode it to BC6H. // (Well, except in fastest mode.) if (m_params.m_uastc_hdr_4x4_options.m_level > 0) @@ -830,12 +1186,12 @@ namespace basisu blk_desc.m_weight_ise_range = best_results.m_best_blk.m_weight_ise_range; blk_desc.m_endpoint_ise_range = best_results.m_best_blk.m_endpoint_ise_range; } - + { std::lock_guard lck(unique_block_desc_mutex); - + auto res = unique_block_descs.insert(std::make_pair(blk_desc, uastc_blk_desc_stats())); - + (res.first)->second.m_count++; #ifdef UASTC_HDR_DEBUG_SAVE_CATEGORIZED_BLOCKS (res.first)->second.m_blks.push_back(dest_block); @@ -843,17 +1199,35 @@ namespace basisu } } - total_blocks_processed++; + } // block_index + + if (m_params.m_status_output) + { + float percent_done = 0; + bool print_flag = false; - uint32_t val = total_blocks_processed; - if (((val & 1023) == 1023) && m_params.m_status_output) { - debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: %3.1f%% done\n", static_cast(val) * 100.0f / total_blocks); + std::lock_guard lck(status_output_mutex); + + total_blocks_processed += (last_index - first_index) + 1; + + percent_done = ((float)total_blocks_processed * 100.0f) / (float)total_blocks; + + if ((percent_done >= 100.0f) || (percent_done >= (last_percentage_printed + 5.0f))) + { + last_percentage_printed = percent_done; + + print_flag = true; + } } + + // minor print race here, doesn't matter + if (print_flag) + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: %3.1f%% done\n", percent_done); } }); - + } // block_index_iter m_params.m_pJob_pool->wait_for_all(); @@ -867,7 +1241,7 @@ namespace basisu m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(tex.get_ptr(), tex.get_size_in_bytes(), 0); } // slice_index - + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_hdr: Total time: %3.3f secs\n", tm.get_elapsed_secs()); if (m_params.m_debug) @@ -914,17 +1288,183 @@ namespace basisu debug_printf(" }\n"); } #endif - + c++; } printf("\n"); - + enc_stats.print(); } return cECSuccess; } + // XUASTC 4x4-12x12 or ASTC 4x4-12x12 + basis_compressor::error_code basis_compressor::encode_slices_to_xuastc_or_astc_ldr() + { + if (m_params.m_debug) + debug_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr\n"); + + m_uastc_slice_textures.resize(m_slice_descs.size()); + + const texture_format tex_fmt = basist::basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + const basist::transcoder_texture_format transcoder_tex_fmt = basist::basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + + uint32_t block_width = 0, block_height = 0; + block_width = basist::basis_get_block_width(transcoder_tex_fmt); + block_height = basist::basis_get_block_height(transcoder_tex_fmt); + +#if defined(_DEBUG) || defined(DEBUG) + // sanity checking + { + uint32_t alt_block_width = 0, alt_block_height = 0; + get_basis_tex_format_block_size(m_fmt_mode, alt_block_width, alt_block_height); + assert((block_width == alt_block_width) && (block_height == alt_block_height)); + } +#endif + + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) + m_uastc_slice_textures[slice_index].init(tex_fmt, m_slice_descs[slice_index].m_orig_width, m_slice_descs[slice_index].m_orig_height); + + m_uastc_backend_output.m_tex_format = m_fmt_mode; + + m_uastc_backend_output.m_etc1s = false; + m_uastc_backend_output.m_srgb = m_params.m_ktx2_and_basis_srgb_transfer_function; + m_uastc_backend_output.m_slice_desc = m_slice_descs; + m_uastc_backend_output.m_slice_image_data.resize(m_slice_descs.size()); + m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); + + astc_ldr::astc_ldr_encode_config cfg; + cfg.m_astc_block_width = block_width; + cfg.m_astc_block_height = block_height; + cfg.m_block_blurring_p1 = m_params.m_xuastc_ldr_blurring; // experimental, not recommended, very slow + cfg.m_block_blurring_p2 = m_params.m_xuastc_ldr_blurring; // experimental, not recommended, very slow + cfg.m_effort_level = clamp(m_params.m_xuastc_ldr_effort_level, astc_ldr::EFFORT_LEVEL_MIN, astc_ldr::EFFORT_LEVEL_MAX); + cfg.m_force_disable_subsets = m_params.m_xuastc_ldr_force_disable_subsets; + cfg.m_force_disable_rgb_dual_plane = m_params.m_xuastc_ldr_force_disable_rgb_dual_plane; + cfg.m_astc_decode_mode_srgb = m_params.m_ktx2_and_basis_srgb_transfer_function; + + cfg.m_compressed_syntax = (basist::astc_ldr_t::xuastc_ldr_syntax)(int)m_params.m_xuastc_ldr_syntax; + if (cfg.m_compressed_syntax >= basist::astc_ldr_t::xuastc_ldr_syntax::cTotal) + { + error_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr: Invalid XUASTC LDR syntax\n"); + return cECFailedInvalidParameters; + } + + if (basist::basis_tex_format_is_xuastc_ldr(m_fmt_mode)) + { + if (m_params.m_quality_level >= 0) + { + // Enable weight grid DCT + cfg.m_dct_quality = static_cast(clamp(m_params.m_quality_level, astc_ldr::DCT_QUALITY_MIN, astc_ldr::DCT_QUALITY_MAX)); + cfg.m_use_dct = m_params.m_xuastc_ldr_use_dct; + } + else + { + // No DCT quality level specified, but they wanted DCT - display warning + if (cfg.m_use_dct) + { + printf("Warning: m_use_dct enabled, but m_quality_level was -1 (not set). Not using DCT. Quality level must range from 1-100.\n"); + } + } + } + + cfg.m_lossy_supercompression = m_params.m_xuastc_ldr_use_lossy_supercompression; + + for (uint32_t i = 0; i < 4; i++) + cfg.m_comp_weights[i] = m_params.m_xuastc_ldr_channel_weights[i]; + + cfg.m_replacement_min_psnr = m_params.m_ls_min_psnr; + cfg.m_psnr_trial_diff_thresh = m_params.m_ls_thresh_psnr; + cfg.m_psnr_trial_diff_thresh_edge = m_params.m_ls_thresh_edge_psnr; + + cfg.m_replacement_min_psnr_alpha = m_params.m_ls_min_alpha_psnr; + cfg.m_psnr_trial_diff_thresh_alpha = m_params.m_ls_thresh_alpha_psnr; + cfg.m_psnr_trial_diff_thresh_edge_alpha = m_params.m_ls_thresh_edge_alpha_psnr; + + cfg.m_debug_output = m_params.m_debug; + cfg.m_debug_images = m_params.m_debug_images; + + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) + { + gpu_image& dst_tex = m_uastc_slice_textures[slice_index]; + + basisu_backend_slice_desc& slice_desc = m_slice_descs[slice_index]; + (void)slice_desc; + + const image& slice_source_image = m_slice_images[slice_index]; + const image* pSource_image = &slice_source_image; + + image temp_image; + if ((slice_source_image.get_width() != slice_desc.m_orig_width) || (slice_source_image.get_height() != slice_desc.m_orig_height)) + { + // Copy to actual/original dimensions so PSNR statistics are calculated correctly. (There's no need to pad the image to multiples of the block dimensions.) + temp_image = slice_source_image; + temp_image.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); + pSource_image = &temp_image; + } + + cfg.m_debug_file_prefix = fmt_string("slice_{}_", slice_index); + + if (m_params.m_debug) + fmt_debug_printf("----------------------------------------------------------------------------\n"); + + uint8_vec intermediate_tex_data; + vector2D coded_log_blocks; + + bool comp_status = astc_ldr::compress_image(*pSource_image, intermediate_tex_data, coded_log_blocks, cfg, *m_params.m_pJob_pool); + if (!comp_status) + return cECFailedEncodeUASTC; + + if (m_params.m_debug) + fmt_debug_printf("----------------------------------------------------------------------------\n"); + + const uint32_t num_blocks_x = dst_tex.get_blocks_x(); + const uint32_t num_blocks_y = dst_tex.get_blocks_y(); + + assert(coded_log_blocks.get_width() == num_blocks_x); + assert(coded_log_blocks.get_height() == num_blocks_y); + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + const astc_helpers::log_astc_block& log_blk = coded_log_blocks(bx, by); + + bool pack_status = astc_helpers::pack_astc_block(*static_cast(dst_tex.get_block_ptr(bx, by)), log_blk); + if (!pack_status) + { + error_printf("basis_compressor::encode_slices_to_xuastc_or_astc_ldr: pack_astc_block() failed!\n"); + return cECFailedEncodeUASTC; + } + + } // bx + } // by + + uint8_vec& dst_buf = m_uastc_backend_output.m_slice_image_data[slice_index]; + + if (basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // Plain ASTC LDR 4x4-12x12 + dst_buf.resize(dst_tex.get_size_in_bytes()); + memcpy(&dst_buf[0], dst_tex.get_ptr(), dst_tex.get_size_in_bytes()); + } + else + { + // Supercompressed XUASTC LDR 4x4-12x12 + assert(intermediate_tex_data.size_in_bytes()); + + dst_buf.resize(intermediate_tex_data.size_in_bytes()); + memcpy(&dst_buf[0], intermediate_tex_data.get_ptr(), intermediate_tex_data.size_in_bytes()); + } + + m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(dst_buf.get_ptr(), dst_buf.size_in_bytes(), 0); + + } // slice_index + + return cECSuccess; + } + basis_compressor::error_code basis_compressor::encode_slices_to_uastc_4x4_ldr() { debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr\n"); @@ -933,12 +1473,12 @@ namespace basisu for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) m_uastc_slice_textures[slice_index].init(texture_format::cUASTC4x4, m_slice_descs[slice_index].m_orig_width, m_slice_descs[slice_index].m_orig_height); - m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC4x4; + m_uastc_backend_output.m_tex_format = basist::basis_tex_format::cUASTC_LDR_4x4; m_uastc_backend_output.m_etc1s = false; m_uastc_backend_output.m_slice_desc = m_slice_descs; m_uastc_backend_output.m_slice_image_data.resize(m_slice_descs.size()); m_uastc_backend_output.m_slice_image_crcs.resize(m_slice_descs.size()); - + for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { gpu_image& tex = m_uastc_slice_textures[slice_index]; @@ -950,19 +1490,21 @@ namespace basisu const uint32_t total_blocks = tex.get_total_blocks(); const image& source_image = m_slice_images[slice_index]; - std::atomic total_blocks_processed; - total_blocks_processed.store(0); - + std::mutex status_output_mutex; + uint32_t total_blocks_processed = 0; + float last_percentage_printed = 0; + const uint32_t N = 256; for (uint32_t block_index_iter = 0; block_index_iter < total_blocks; block_index_iter += N) { const uint32_t first_index = block_index_iter; const uint32_t last_index = minimum(total_blocks, block_index_iter + N); - - m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, &tex, &total_blocks_processed] + + m_params.m_pJob_pool->add_job([this, first_index, last_index, num_blocks_x, num_blocks_y, total_blocks, &source_image, &tex, + &status_output_mutex, &total_blocks_processed, &last_percentage_printed] { BASISU_NOTE_UNUSED(num_blocks_y); - + uint32_t uastc_flags = m_params.m_pack_uastc_ldr_4x4_flags; if ((m_params.m_rdo_uastc_ldr_4x4) && (m_params.m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode)) uastc_flags |= cPackUASTCFavorSimplerModes; @@ -980,14 +1522,31 @@ namespace basisu encode_uastc(&block_pixels[0][0].r, dest_block, uastc_flags); - total_blocks_processed++; + } // block_index + + if (m_params.m_status_output) + { + float percent_done = 0; + bool print_flag = false; - uint32_t val = total_blocks_processed; - if (((val & 16383) == 16383) && m_params.m_status_output) { - debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr: %3.1f%% done\n", static_cast(val) * 100.0f / total_blocks); + std::lock_guard lck(status_output_mutex); + + total_blocks_processed += (last_index - first_index) + 1; + + percent_done = ((float)total_blocks_processed * 100.0f) / (float)total_blocks; + + if ((percent_done >= 100.0f) || (percent_done >= (last_percentage_printed + 5.0f))) + { + last_percentage_printed = percent_done; + + print_flag = true; + } } + // minor print race here, doesn't matter + if (print_flag) + debug_printf("basis_compressor::encode_slices_to_uastc_4x4_ldr: %3.1f%% done\n", percent_done); } }); @@ -1005,7 +1564,7 @@ namespace basisu rdo_params.m_lz_dict_size = m_params.m_rdo_uastc_ldr_4x4_dict_size; rdo_params.m_smooth_block_max_error_scale = m_params.m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale; rdo_params.m_max_smooth_block_std_dev = m_params.m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev; - + bool status = uastc_rdo(tex.get_total_blocks(), (basist::uastc_block*)tex.get_ptr(), (const color_rgba *)m_source_blocks[slice_desc.m_first_block_index].m_pixels, rdo_params, m_params.m_pack_uastc_ldr_4x4_flags, m_params.m_rdo_uastc_ldr_4x4_multithreading ? m_params.m_pJob_pool : nullptr, (m_params.m_rdo_uastc_ldr_4x4_multithreading && m_params.m_pJob_pool) ? basisu::minimum(4, (uint32_t)m_params.m_pJob_pool->get_total_threads()) : 0); @@ -1017,11 +1576,11 @@ namespace basisu m_uastc_backend_output.m_slice_image_data[slice_index].resize(tex.get_size_in_bytes()); memcpy(&m_uastc_backend_output.m_slice_image_data[slice_index][0], tex.get_ptr(), tex.get_size_in_bytes()); - + m_uastc_backend_output.m_slice_image_crcs[slice_index] = basist::crc16(tex.get_ptr(), tex.get_size_in_bytes(), 0); - + } // slice_index - + return cECSuccess; } @@ -1057,8 +1616,8 @@ namespace basisu pSource_image = &mips[level - 1]; } - bool status = image_resample(*pSource_image, level_img, - //m_params.m_mip_filter.c_str(), + bool status = image_resample(*pSource_image, level_img, + //m_params.m_mip_filter.c_str(), "box", // TODO: negative lobes in the filter are causing negative colors, try Mitchell m_params.m_mip_scale, m_params.m_mip_wrapping, 0, has_alpha ? 4 : 3); if (!status) @@ -1113,12 +1672,12 @@ namespace basisu image &level_img = *enlarge_vector(mips, 1); level_img.resize(level_width, level_height); - - int result = stbir_resize_uint8_generic( + + int result = stbir_resize_uint8_generic( (const uint8_t *)img.get_ptr(), img.get_width(), img.get_height(), img.get_pitch() * sizeof(color_rgba), (uint8_t *)level_img.get_ptr(), level_img.get_width(), level_img.get_height(), level_img.get_pitch() * sizeof(color_rgba), has_alpha ? 4 : 3, has_alpha ? 3 : STBIR_ALPHA_CHANNEL_NONE, m_params.m_mip_premultiplied ? STBIR_FLAG_ALPHA_PREMULTIPLIED : 0, - m_params.m_mip_wrapping ? STBIR_EDGE_WRAP : STBIR_EDGE_CLAMP, filter, m_params.m_mip_srgb ? STBIR_COLORSPACE_SRGB : STBIR_COLORSPACE_LINEAR, + m_params.m_mip_wrapping ? STBIR_EDGE_WRAP : STBIR_EDGE_CLAMP, filter, m_params.m_mip_srgb ? STBIR_COLORSPACE_SRGB : STBIR_COLORSPACE_LINEAR, nullptr); if (result == 0) @@ -1126,7 +1685,7 @@ namespace basisu error_printf("basis_compressor::generate_mipmaps: stbir_resize_uint8_generic() failed!\n"); return false; } - + if (m_params.m_mip_renormalize) level_img.renormalize_normal_map(); } @@ -1250,7 +1809,7 @@ namespace basisu if (m_params.m_source_mipmap_images.size() || m_params.m_source_mipmap_images_hdr.size()) return true; - + // See if any input filenames are .DDS bool any_dds = false, all_dds = true; for (uint32_t i = 0; i < m_params.m_source_filenames.size(); i++) @@ -1319,7 +1878,7 @@ namespace basisu ldr_mips.erase_index(0U); m_params.m_source_mipmap_images.back().swap(ldr_mips); - + any_mipmaps = true; } } @@ -1339,7 +1898,7 @@ namespace basisu hdr_mips.erase_index(0U); m_params.m_source_mipmap_images_hdr.back().swap(hdr_mips); - + any_mipmaps = true; } @@ -1363,7 +1922,7 @@ namespace basisu error_printf("HDR mode enabled, but only LDR .DDS files were loaded. HDR mode requires half or float (HDR) .DDS inputs.\n"); return false; } - + return true; } @@ -1371,7 +1930,7 @@ namespace basisu { debug_printf("basis_compressor::read_source_images\n"); - const uint32_t total_source_files = m_params.m_read_source_images ? (uint32_t)m_params.m_source_filenames.size() : + const uint32_t total_source_files = m_params.m_read_source_images ? (uint32_t)m_params.m_source_filenames.size() : (m_params.m_hdr ? (uint32_t)m_params.m_source_images_hdr.size() : (uint32_t)m_params.m_source_images.size()); if (!total_source_files) @@ -1395,7 +1954,7 @@ namespace basisu basisu::vector source_images_hdr; basisu::vector source_filenames; - + // TODO: Note HDR images don't support alpha here, currently. // First load all source images, and determine if any have an alpha channel. @@ -1530,7 +2089,7 @@ namespace basisu for (uint32_t x = 0; x < file_image_hdr.get_width(); x++) { const vec4F& c = file_image_hdr(x, y); - + // For now, alpha is always 1.0f in UASTC HDR. file_image_hdr(x, y).set(c[m_params.m_swizzle[0]], c[m_params.m_swizzle[1]], c[m_params.m_swizzle[2]], 1.0f); // c[m_params.m_swizzle[3]]); } @@ -1648,7 +2207,7 @@ namespace basisu source_filenames.push_back(pSource_filename); } - // Check if the caller has generated their own mipmaps. + // Check if the caller has generated their own mipmaps. if (m_params.m_hdr) { if (m_params.m_source_mipmap_images_hdr.size()) @@ -1661,7 +2220,7 @@ namespace basisu } } } - else + else { if (m_params.m_source_mipmap_images.size()) { @@ -1702,23 +2261,23 @@ namespace basisu for (uint32_t source_file_index = 0; source_file_index < total_source_files; source_file_index++) { const std::string &source_filename = source_filenames[source_file_index]; - + basisu::vector slices; basisu::vector slices_hdr; - + slices.reserve(32); slices_hdr.reserve(32); - + // The first (largest) mipmap level. image *pFile_image = source_images.size() ? &source_images[source_file_index] : nullptr; imagef *pFile_image_hdr = source_images_hdr.size() ? &source_images_hdr[source_file_index] : nullptr; - + // Reserve a slot for mip0. if (m_params.m_hdr) slices_hdr.resize(1); else slices.resize(1); - + if ((!m_params.m_hdr) && (m_params.m_source_mipmap_images.size())) { // User-provided mipmaps for each layer or image in the texture array. @@ -1800,10 +2359,10 @@ namespace basisu uint_vec mip_indices(m_params.m_hdr ? slices_hdr.size() : slices.size()); for (uint32_t i = 0; i < (m_params.m_hdr ? slices_hdr.size() : slices.size()); i++) mip_indices[i] = i; - + if ((!m_params.m_hdr) && (m_any_source_image_has_alpha) && (!m_params.m_uastc)) { - // For ETC1S, if source has alpha, then even mips will have RGB, and odd mips will have alpha in RGB. + // For ETC1S, if source has alpha, then even mips will have RGB, and odd mips will have alpha in RGB. basisu::vector alpha_slices; uint_vec new_mip_indices; @@ -1822,7 +2381,7 @@ namespace basisu lvl_a(x, y).set_noclamp_rgba(a, a, a, 255); } } - + lvl_rgb.set_alpha(255); alpha_slices.push_back(lvl_rgb); @@ -1844,7 +2403,7 @@ namespace basisu { assert(slices.size() == mip_indices.size()); } - + for (uint32_t slice_index = 0; slice_index < (m_params.m_hdr ? slices_hdr.size() : slices.size()); slice_index++) { image *pSlice_image = m_params.m_hdr ? nullptr : &slices[slice_index]; @@ -1903,10 +2462,10 @@ namespace basisu m_stats[dest_image_index].m_width = orig_width; m_stats[dest_image_index].m_height = orig_height; - debug_printf("****** Slice %u: mip %u, alpha_slice: %u, filename: \"%s\", original: %ux%u actual: %ux%u\n", - m_slice_descs.size() - 1, mip_indices[slice_index], is_alpha_slice, source_filename.c_str(), - orig_width, orig_height, - m_params.m_hdr ? pSlice_image_hdr->get_width() : pSlice_image->get_width(), + debug_printf("****** Slice %u: mip %u, alpha_slice: %u, filename: \"%s\", original: %ux%u actual: %ux%u\n", + m_slice_descs.size() - 1, mip_indices[slice_index], is_alpha_slice, source_filename.c_str(), + orig_width, orig_height, + m_params.m_hdr ? pSlice_image_hdr->get_width() : pSlice_image->get_width(), m_params.m_hdr ? pSlice_image_hdr->get_height() : pSlice_image->get_height()); basisu_backend_slice_desc& slice_desc = m_slice_descs[dest_image_index]; @@ -1969,7 +2528,7 @@ namespace basisu error_printf("Too many slices!\n"); return false; } - + // Basic sanity check on the slices for (uint32_t i = 1; i < m_slice_descs.size(); i++) { @@ -1979,7 +2538,7 @@ namespace basisu // Make sure images are in order int image_delta = (int)slice_desc.m_source_file_index - (int)prev_slice_desc.m_source_file_index; if (image_delta > 1) - return false; + return false; // Make sure mipmap levels are in order if (!image_delta) @@ -2002,8 +2561,8 @@ namespace basisu if (m_params.m_status_output) { printf("Slice: %u, alpha: %u, orig width/height: %ux%u, width/height: %ux%u, first_block: %u, image_index: %u, mip_level: %u, iframe: %u\n", - i, slice_desc.m_alpha, slice_desc.m_orig_width, slice_desc.m_orig_height, - slice_desc.m_width, slice_desc.m_height, + i, slice_desc.m_alpha, slice_desc.m_orig_width, slice_desc.m_orig_height, + slice_desc.m_width, slice_desc.m_height, slice_desc.m_first_block_index, slice_desc.m_source_file_index, slice_desc.m_mip_index, slice_desc.m_iframe); } @@ -2058,20 +2617,20 @@ namespace basisu } // Do some basic validation for 2D arrays, cubemaps, video, and volumes. - bool basis_compressor::validate_texture_type_constraints() + bool basis_compressor::validate_texture_type_constraints() { debug_printf("basis_compressor::validate_texture_type_constraints\n"); // In 2D mode anything goes (each image may have a different resolution and # of mipmap levels). if (m_params.m_tex_type == basist::cBASISTexType2D) return true; - + uint32_t total_basis_images = 0; for (uint32_t slice_index = 0; slice_index < (m_params.m_hdr ? m_slice_images_hdr.size() : m_slice_images.size()); slice_index++) { const basisu_backend_slice_desc &slice_desc = m_slice_descs[slice_index]; - + total_basis_images = maximum(total_basis_images, slice_desc.m_source_file_index + 1); } @@ -2094,7 +2653,7 @@ namespace basisu const basisu_backend_slice_desc &slice_desc = m_slice_descs[slice_index]; image_mipmap_levels[slice_desc.m_source_file_index] = maximum(image_mipmap_levels[slice_desc.m_source_file_index], slice_desc.m_mip_index + 1); - + if (slice_desc.m_mip_index != 0) continue; @@ -2127,9 +2686,13 @@ namespace basisu debug_printf("basis_compressor::extract_source_blocks\n"); // No need to extract blocks in 6x6 mode, but the 4x4 compressors want 4x4 blocks. - if ((m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6) || (m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + if ((m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6) || (m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) return true; + // No need to extract blocks in XUASTC/ASTC LDR mode either. + if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + return true; + if (m_params.m_hdr) m_source_blocks_hdr.resize(m_total_blocks); else @@ -2184,7 +2747,7 @@ namespace basisu bool basis_compressor::process_frontend() { debug_printf("basis_compressor::process_frontend\n"); - + #if 0 // TODO basis_etc1_pack_params pack_params; @@ -2235,21 +2798,21 @@ namespace basisu error_printf("Too many selector clusters! (%u but max is %u)\n", selector_clusters, basisu_frontend::cMaxSelectorClusters); return false; } - - if (m_params.m_etc1s_quality_level != -1) + + if (m_params.m_quality_level != -1) { - const float quality = saturate(m_params.m_etc1s_quality_level / 255.0f); - + const float quality = saturate(m_params.m_quality_level / 255.0f); + const float bits_per_endpoint_cluster = 14.0f; const float max_desired_endpoint_cluster_bits_per_texel = 1.0f; // .15f int max_endpoints = static_cast((max_desired_endpoint_cluster_bits_per_texel * total_texels) / bits_per_endpoint_cluster); - + const float mid = 128.0f / 255.0f; float color_endpoint_quality = quality; const float endpoint_split_point = 0.5f; - + // In v1.2 and in previous versions, the endpoint codebook size at quality 128 was 3072. This wasn't quite large enough. const int ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE = 4800; const int MAX_ENDPOINT_CODEBOOK_SIZE = 8192; @@ -2260,7 +2823,7 @@ namespace basisu max_endpoints = clamp(max_endpoints, 256, ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE); max_endpoints = minimum(max_endpoints, m_total_blocks); - + if (max_endpoints < 64) max_endpoints = 64; endpoint_clusters = clamp((uint32_t)(.5f + lerp(32, static_cast(max_endpoints), color_endpoint_quality)), 32, basisu_frontend::cMaxEndpointClusters); @@ -2271,12 +2834,12 @@ namespace basisu max_endpoints = clamp(max_endpoints, 256, MAX_ENDPOINT_CODEBOOK_SIZE); max_endpoints = minimum(max_endpoints, m_total_blocks); - + if (max_endpoints < ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE) max_endpoints = ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE; endpoint_clusters = clamp((uint32_t)(.5f + lerp(ENDPOINT_CODEBOOK_MID_QUALITY_CODEBOOK_SIZE, static_cast(max_endpoints), color_endpoint_quality)), 32, basisu_frontend::cMaxEndpointClusters); } - + float bits_per_selector_cluster = 14.0f; const float max_desired_selector_cluster_bits_per_texel = 1.0f; // .15f @@ -2294,18 +2857,18 @@ namespace basisu debug_printf("Max endpoints: %u, max selectors: %u\n", endpoint_clusters, selector_clusters); - if (m_params.m_etc1s_quality_level >= 223) + if (m_params.m_quality_level >= 223) { if (!m_params.m_selector_rdo_thresh.was_changed()) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .25f; - + if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .25f; } } - else if (m_params.m_etc1s_quality_level >= 192) + else if (m_params.m_quality_level >= 192) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .5f; @@ -2313,7 +2876,7 @@ namespace basisu if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .5f; } - else if (m_params.m_etc1s_quality_level >= 160) + else if (m_params.m_quality_level >= 160) { if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= .75f; @@ -2321,18 +2884,18 @@ namespace basisu if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= .75f; } - else if (m_params.m_etc1s_quality_level >= 129) + else if (m_params.m_quality_level >= 129) { float l = (quality - 129 / 255.0f) / ((160 - 129) / 255.0f); if (!m_params.m_endpoint_rdo_thresh.was_changed()) m_params.m_endpoint_rdo_thresh *= lerp(1.0f, .75f, l); - + if (!m_params.m_selector_rdo_thresh.was_changed()) m_params.m_selector_rdo_thresh *= lerp(1.0f, .75f, l); } } - + basisu_frontend::params p; p.m_num_source_blocks = m_total_blocks; p.m_pSource_blocks = &m_source_blocks[0]; @@ -2341,14 +2904,14 @@ namespace basisu p.m_perceptual = m_params.m_perceptual; p.m_debug_stats = m_params.m_debug; p.m_debug_images = m_params.m_debug_images; - p.m_compression_level = m_params.m_compression_level; + p.m_compression_level = m_params.m_etc1s_compression_level; p.m_tex_type = m_params.m_tex_type; p.m_multithreaded = m_params.m_multithreading; p.m_disable_hierarchical_endpoint_codebooks = m_params.m_disable_hierarchical_endpoint_codebooks; p.m_validate = m_params.m_validate_etc1s; p.m_pJob_pool = m_params.m_pJob_pool; p.m_pGlobal_codebooks = m_params.m_pGlobal_codebooks; - + // Don't keep trying to use OpenCL if it ever fails. p.m_pOpenCL_context = !m_opencl_failed ? m_pOpenCL_context : nullptr; @@ -2357,7 +2920,7 @@ namespace basisu error_printf("basisu_frontend::init() failed!\n"); return false; } - + m_frontend.compress(); if (m_frontend.get_opencl_failed()) @@ -2368,18 +2931,18 @@ namespace basisu for (uint32_t i = 0; i < m_slice_descs.size(); i++) { char filename[1024]; -#ifdef _WIN32 +#ifdef _WIN32 sprintf_s(filename, sizeof(filename), "rdo_frontend_output_output_blocks_%u.png", i); #else snprintf(filename, sizeof(filename), "rdo_frontend_output_output_blocks_%u.png", i); -#endif +#endif m_frontend.dump_debug_image(filename, m_slice_descs[i].m_first_block_index, m_slice_descs[i].m_num_blocks_x, m_slice_descs[i].m_num_blocks_y, true); #ifdef _WIN32 sprintf_s(filename, sizeof(filename), "rdo_frontend_output_api_%u.png", i); #else snprintf(filename, sizeof(filename), "rdo_frontend_output_api_%u.png", i); -#endif +#endif m_frontend.dump_debug_image(filename, m_slice_descs[i].m_first_block_index, m_slice_descs[i].m_num_blocks_x, m_slice_descs[i].m_num_blocks_y, false); } } @@ -2428,7 +2991,7 @@ namespace basisu for (uint32_t block_x = 0; block_x < num_blocks_x; block_x++) memcpy(m_best_etc1s_images[i].get_block_ptr(block_x, block_y, 0), &m_frontend.get_etc1s_block(slice_desc.m_first_block_index + block_x + block_y * num_blocks_x), sizeof(etc_block)); - m_best_etc1s_images[i].unpack(m_best_etc1s_images_unpacked[i]); + m_best_etc1s_images[i].unpack(m_best_etc1s_images_unpacked[i], false); } return true; @@ -2442,7 +3005,7 @@ namespace basisu backend_params.m_debug = m_params.m_debug; backend_params.m_debug_images = m_params.m_debug_images; backend_params.m_etc1s = true; - backend_params.m_compression_level = m_params.m_compression_level; + backend_params.m_compression_level = m_params.m_etc1s_compression_level; if (!m_params.m_no_endpoint_rdo) backend_params.m_endpoint_rdo_quality_thresh = m_params.m_endpoint_rdo_thresh; @@ -2484,7 +3047,7 @@ namespace basisu m_output_basis_file = comp_data; uint32_t total_orig_pixels = 0; - + for (uint32_t i = 0; i < m_slice_descs.size(); i++) { const basisu_backend_slice_desc& slice_desc = m_slice_descs[i]; @@ -2492,22 +3055,22 @@ namespace basisu total_orig_pixels += slice_desc.m_orig_width * slice_desc.m_orig_height; } - m_basis_file_size = (uint32_t)comp_data.size(); + m_total_slice_orig_texels = total_orig_pixels; + m_basis_file_size = comp_data.size(); m_basis_bits_per_texel = total_orig_pixels ? (comp_data.size() * 8.0f) / total_orig_pixels : 0; - debug_printf("Total .basis output file size: %u, %3.3f bits/texel\n", comp_data.size(), comp_data.size() * 8.0f / total_orig_pixels); + fmt_debug_printf("Total .basis output file size: {}, {3.3} bits/texel\n", m_basis_file_size, m_basis_bits_per_texel); // HDR 6x6 TODO - // HACK HACK const bool is_hdr_6x6 = m_params.m_hdr && (m_params.m_hdr_mode != hdr_modes::cUASTC_HDR_4X4); - + if (m_params.m_validate_output_data) { interval_timer tm; tm.start(); basist::basisu_transcoder_init(); - + debug_printf("basist::basisu_transcoder_init: Took %f ms\n", tm.get_elapsed_ms()); // Verify the compressed data by transcoding it to ASTC (or ETC1)/BC7 and validating the CRC's. @@ -2554,29 +3117,56 @@ namespace basisu double total_time_etc1s_or_astc = 0; + // Select formats to transcode to + basisu::texture_format tex_format; + basist::block_format blk_format; + + if (m_params.m_hdr) + { + // HDR + tex_format = texture_format::cBC6HUnsigned; + blk_format = basist::block_format::cBC6H; + } + else if (m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) + { + // UASTC LDR 4x4 + tex_format = texture_format::cUASTC4x4; + blk_format = basist::block_format::cUASTC_4x4; + } + else if (basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode)) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + basist::transcoder_texture_format transcoder_fmt = basist::basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + + tex_format = basist::basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(m_fmt_mode); + blk_format = basist::xuastc_get_block_format(transcoder_fmt); + } + else + { + // ETC1S + tex_format = texture_format::cETC1; + blk_format = basist::block_format::cETC1; + } + for (uint32_t slice_iter = 0; slice_iter < m_slice_descs.size(); slice_iter++) { - // Select either BC6H, UASTC LDR 4x4, or ETC1 - basisu::texture_format tex_format = m_params.m_hdr ? texture_format::cBC6HUnsigned : (m_params.m_uastc ? texture_format::cUASTC4x4 : texture_format::cETC1); - basist::block_format blk_format = m_params.m_hdr ? basist::block_format::cBC6H : (m_params.m_uastc ? basist::block_format::cUASTC_4x4 : basist::block_format::cETC1); - gpu_image decoded_texture; decoded_texture.init( tex_format, - m_slice_descs[slice_iter].m_width, m_slice_descs[slice_iter].m_height); + m_slice_descs[slice_iter].m_orig_width, m_slice_descs[slice_iter].m_orig_height); + + const uint32_t dst_block_size_x = basisu::get_block_width(tex_format); + const uint32_t dst_block_size_y = basisu::get_block_height(tex_format); + const uint32_t num_dst_blocks_x = (m_slice_descs[slice_iter].m_orig_width + dst_block_size_x - 1) / dst_block_size_x; + const uint32_t num_dst_blocks_y = (m_slice_descs[slice_iter].m_orig_height + dst_block_size_y - 1) / dst_block_size_y; + const uint32_t total_dst_blocks = num_dst_blocks_x * num_dst_blocks_y; + + const uint32_t bytes_per_block = decoded_texture.get_bytes_per_block(); tm.start(); - const uint32_t block_size_x = basisu::get_block_width(tex_format); - const uint32_t block_size_y = basisu::get_block_height(tex_format); - const uint32_t num_dst_blocks_x = (m_slice_descs[slice_iter].m_orig_width + block_size_x - 1) / block_size_x; - const uint32_t num_dst_blocks_y = (m_slice_descs[slice_iter].m_orig_height + block_size_y - 1) / block_size_y; - const uint32_t total_dst_blocks = num_dst_blocks_x * num_dst_blocks_y; - - uint32_t bytes_per_block = m_params.m_uastc ? 16 : 8; - if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), slice_iter, - reinterpret_cast(decoded_texture.get_ptr()), total_dst_blocks, blk_format, bytes_per_block)) + reinterpret_cast(decoded_texture.get_ptr()), total_dst_blocks, blk_format, bytes_per_block, m_params.m_transcode_flags)) { error_printf("Transcoding failed on slice %u!\n", slice_iter); return false; @@ -2606,15 +3196,15 @@ namespace basisu if (is_hdr_6x6) { assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cASTC_HDR_6x6)); - assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)); + assert(basist::basis_is_format_supported(basist::transcoder_texture_format::cTFASTC_HDR_6x6_RGBA, basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)); for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cASTC_HDR_6x6, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cASTC_HDR_6x6, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_6x6, 16)) + reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_6x6, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to ASTC HDR on slice %u!\n", i); return false; @@ -2630,10 +3220,10 @@ namespace basisu for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cASTC_HDR_4x4, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cASTC_HDR_4x4, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_4x4, 16)) + reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cASTC_HDR_4x4, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to ASTC HDR on slice %u!\n", i); return false; @@ -2645,16 +3235,19 @@ namespace basisu } else { - if (basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cUASTC4x4) && + if (basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cUASTC_LDR_4x4) && basist::basis_is_format_supported(basist::transcoder_texture_format::cTFBC7_RGBA, basist::basis_tex_format::cETC1S)) { for (uint32_t i = 0; i < m_slice_descs.size(); i++) { gpu_image decoded_texture; - decoded_texture.init(texture_format::cBC7, m_slice_descs[i].m_width, m_slice_descs[i].m_height); + decoded_texture.init(texture_format::cBC7, m_slice_descs[i].m_orig_width, m_slice_descs[i].m_orig_height); + + const uint32_t num_bc7_blocks_x = decoded_texture.get_blocks_x(); + const uint32_t num_bc7_blocks_y = decoded_texture.get_blocks_y(); if (!decoder.transcode_slice(&comp_data[0], (uint32_t)comp_data.size(), i, - reinterpret_cast(decoded_texture.get_ptr()), m_slice_descs[i].m_num_blocks_x * m_slice_descs[i].m_num_blocks_y, basist::block_format::cBC7, 16)) + decoded_texture.get_ptr(), num_bc7_blocks_x * num_bc7_blocks_y, basist::block_format::cBC7, 16, m_params.m_transcode_flags)) { error_printf("Transcoding failed to BC7 on slice %u!\n", i); return false; @@ -2671,25 +3264,37 @@ namespace basisu { if (m_params.m_hdr) { - // BC6H bool status = m_decoded_output_textures[i].unpack_hdr(m_decoded_output_textures_bc6h_hdr_unpacked[i]); - assert(status); - BASISU_NOTE_UNUSED(status); - - // ASTC HDR + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } + status = m_decoded_output_textures_astc_hdr[i].unpack_hdr(m_decoded_output_textures_astc_hdr_unpacked[i]); - assert(status); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } } else { - bool status = m_decoded_output_textures[i].unpack(m_decoded_output_textures_unpacked[i]); - assert(status); - BASISU_NOTE_UNUSED(status); + bool status = m_decoded_output_textures[i].unpack(m_decoded_output_textures_unpacked[i], m_params.m_ktx2_and_basis_srgb_transfer_function); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } if (m_decoded_output_textures_bc7[i].get_pixel_width()) { - status = m_decoded_output_textures_bc7[i].unpack(m_decoded_output_textures_unpacked_bc7[i]); - assert(status); + status = m_decoded_output_textures_bc7[i].unpack(m_decoded_output_textures_unpacked_bc7[i], m_params.m_ktx2_and_basis_srgb_transfer_function); + if (!status) + { + error_printf("Unpacking failed on slice %u!\n", i); + return false; + } } } } @@ -2703,6 +3308,7 @@ namespace basisu if (!is_hdr_6x6) { + // Sanity check decoded output texture sizes for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { const basisu_backend_slice_desc& slice_desc = m_slice_descs[slice_index]; @@ -2715,7 +3321,7 @@ namespace basisu } } // if (m_params.m_validate_output_data) - + return true; } @@ -2756,7 +3362,7 @@ namespace basisu { const std::string filename(string_format("%s_compressive_tonemapped.png", pBasename)); image compressive_tonemapped_img; - + bool status = tonemap_image_compressive(compressive_tonemapped_img, hdr_img); if (!status) { @@ -2802,7 +3408,7 @@ namespace basisu if (m_params.m_status_output) { - printf("Wrote output .basis/.ktx2 file \"%s\"\n", output_filename.c_str()); + printf("Wrote compressed output file \"%s\"\n", output_filename.c_str()); } } @@ -2824,7 +3430,7 @@ namespace basisu uint32_t total_texels = 0; for (uint32_t i = 0; i < m_slice_descs.size(); i++) total_texels += (m_slice_descs[i].m_orig_width * m_slice_descs[i].m_orig_height); - + m_basis_bits_per_texel = ((float)comp_size * 8.0f) / total_texels; fmt_debug_printf("Output file size: {}, {3.2} bits/texel, LZ compressed file size: {}, {3.2} bits/texel\n", @@ -2833,7 +3439,7 @@ namespace basisu } m_stats.resize(m_slice_descs.size()); - + if (m_params.m_validate_output_data) { if (m_params.m_hdr) @@ -2906,7 +3512,7 @@ namespace basisu im.calc(m_slice_images_hdr[slice_index], m_decoded_output_textures_astc_hdr_unpacked[slice_index], 0, 3, true, true); s.m_basis_rgb_avg_log2_psnr = (float)im.m_psnr; - + if (m_params.m_print_stats) { printf("\nASTC Log2 RGB: "); @@ -2924,7 +3530,7 @@ namespace basisu printf("\n"); } } - + if (m_params.m_debug_images) { std::string out_basename; @@ -2940,9 +3546,9 @@ namespace basisu { gpu_image bc6h_tex(m_decoded_output_textures[slice_index]); bc6h_tex.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename(out_basename + "_bc6h.dds"); - write_compressed_texture_file(filename.c_str(), bc6h_tex, true); + write_compressed_texture_file(filename.c_str(), bc6h_tex, false); printf("Wrote .DDS file %s\n", filename.c_str()); } @@ -2950,11 +3556,11 @@ namespace basisu { gpu_image astc_tex(m_decoded_output_textures_astc_hdr[slice_index]); astc_tex.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename1(out_basename + "_astc.astc"); - + uint32_t block_width = 4, block_height = 4; - if ((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE)) + if ((m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6) || (m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE)) { block_width = 6; block_height = 6; @@ -2964,7 +3570,7 @@ namespace basisu printf("Wrote .ASTC file %s\n", filename1.c_str()); std::string filename2(out_basename + "_astc.ktx"); - write_compressed_texture_file(filename2.c_str(), astc_tex, true); + write_compressed_texture_file(filename2.c_str(), astc_tex, false); printf("Wrote .KTX file %s\n", filename2.c_str()); } @@ -2972,7 +3578,7 @@ namespace basisu { imagef astc_img(m_decoded_output_textures_astc_hdr_unpacked[slice_index]); astc_img.resize(slice_desc.m_orig_width, slice_desc.m_orig_height); - + std::string filename(out_basename + "_unpacked_astc.exr"); write_exr(filename.c_str(), astc_img, 3, 0); printf("Wrote .EXR file %s\n", filename.c_str()); @@ -3007,50 +3613,50 @@ namespace basisu printf("Slice: %u\n", slice_index); image_stats& s = m_stats[slice_index]; - + image_metrics em; // ---- .basis stats em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 3); if (m_params.m_print_stats) - em.print(".basis RGB Avg: "); + em.print("RGB Avg: "); s.m_basis_rgb_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 4); if (m_params.m_print_stats) - em.print(".basis RGBA Avg: "); + em.print("RGBA Avg: "); s.m_basis_rgba_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 1); if (m_params.m_print_stats) - em.print(".basis R Avg: "); + em.print("R Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 1, 1); if (m_params.m_print_stats) - em.print(".basis G Avg: "); + em.print("G Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 2, 1); if (m_params.m_print_stats) - em.print(".basis B Avg: "); + em.print("B Avg: "); - if (m_params.m_uastc) + //if (m_params.m_uastc) { em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 3, 1); if (m_params.m_print_stats) - em.print(".basis A Avg: "); + em.print("A Avg: "); s.m_basis_a_avg_psnr = (float)em.m_psnr; } em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 0); if (m_params.m_print_stats) - em.print(".basis 709 Luma: "); + em.print("709 Luma: "); s.m_basis_luma_709_psnr = static_cast(em.m_psnr); s.m_basis_luma_709_ssim = static_cast(em.m_ssim); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked[slice_index], 0, 0, true, true); if (m_params.m_print_stats) - em.print(".basis 601 Luma: "); + em.print("601 Luma: "); s.m_basis_luma_601_psnr = static_cast(em.m_psnr); if (m_slice_descs.size() == 1) @@ -3058,8 +3664,8 @@ namespace basisu const uint32_t output_size = comp_size ? (uint32_t)comp_size : (uint32_t)comp_data.size(); if (m_params.m_print_stats) { - debug_printf(".basis RGB PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_rgb_avg_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); - debug_printf(".basis Luma 709 PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_luma_709_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); + debug_printf("RGB PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_rgb_avg_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); + debug_printf("Luma 709 PSNR per bit/texel*10000: %3.3f\n", 10000.0f * s.m_basis_luma_709_psnr / ((output_size * 8.0f) / (slice_desc.m_orig_width * slice_desc.m_orig_height))); } } @@ -3067,45 +3673,45 @@ namespace basisu { // ---- BC7 stats em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 3); - //if (m_params.m_print_stats) - // em.print("BC7 RGB Avg: "); + if (m_params.m_print_stats) + em.print("BC7 RGB Avg: "); s.m_bc7_rgb_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 4); - //if (m_params.m_print_stats) - // em.print("BC7 RGBA Avg: "); + if (m_params.m_print_stats) + em.print("BC7 RGBA Avg: "); s.m_bc7_rgba_avg_psnr = (float)em.m_psnr; em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 1); - //if (m_params.m_print_stats) - // em.print("BC7 R Avg: "); + if (m_params.m_print_stats) + em.print("BC7 R Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 1, 1); - //if (m_params.m_print_stats) - // em.print("BC7 G Avg: "); + if (m_params.m_print_stats) + em.print("BC7 G Avg: "); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 2, 1); - //if (m_params.m_print_stats) - // em.print("BC7 B Avg: "); + if (m_params.m_print_stats) + em.print("BC7 B Avg: "); - if (m_params.m_uastc) + //if (m_params.m_uastc) { em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 3, 1); - //if (m_params.m_print_stats) - // em.print("BC7 A Avg: "); + if (m_params.m_print_stats) + em.print("BC7 A Avg: "); s.m_bc7_a_avg_psnr = (float)em.m_psnr; } em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 0); - //if (m_params.m_print_stats) - // em.print("BC7 709 Luma: "); + if (m_params.m_print_stats) + em.print("BC7 709 Luma: "); s.m_bc7_luma_709_psnr = static_cast(em.m_psnr); s.m_bc7_luma_709_ssim = static_cast(em.m_ssim); em.calc(m_slice_images[slice_index], m_decoded_output_textures_unpacked_bc7[slice_index], 0, 0, true, true); - //if (m_params.m_print_stats) - // em.print("BC7 601 Luma: "); + if (m_params.m_print_stats) + em.print("BC7 601 Luma: "); s.m_bc7_luma_601_psnr = static_cast(em.m_psnr); } @@ -3146,10 +3752,10 @@ namespace basisu { gpu_image best_etc1s_gpu_image(m_best_etc1s_images[slice_index]); best_etc1s_gpu_image.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_best_etc1s.ktx").c_str(), best_etc1s_gpu_image, true); + write_compressed_texture_file((out_basename + "_best_etc1s.ktx").c_str(), best_etc1s_gpu_image, m_params.m_ktx2_and_basis_srgb_transfer_function); image best_etc1s_unpacked; - best_etc1s_gpu_image.unpack(best_etc1s_unpacked); + best_etc1s_gpu_image.unpack(best_etc1s_unpacked, m_params.m_ktx2_and_basis_srgb_transfer_function); save_png(out_basename + "_best_etc1s.png", best_etc1s_unpacked); } } @@ -3160,7 +3766,7 @@ namespace basisu { gpu_image decoded_etc1s_or_astc(m_decoded_output_textures[slice_index]); decoded_etc1s_or_astc.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_transcoded_etc1s_or_astc.ktx").c_str(), decoded_etc1s_or_astc, true); + write_compressed_texture_file((out_basename + "_transcoded_etc1s_or_astc.ktx").c_str(), decoded_etc1s_or_astc, m_params.m_ktx2_and_basis_srgb_transfer_function); image temp(m_decoded_output_textures_unpacked[slice_index]); temp.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); @@ -3172,21 +3778,28 @@ namespace basisu { gpu_image decoded_bc7(m_decoded_output_textures_bc7[slice_index]); decoded_bc7.override_dimensions(slice_desc.m_orig_width, slice_desc.m_orig_height); - write_compressed_texture_file((out_basename + "_transcoded_bc7.ktx").c_str(), decoded_bc7, true); + write_compressed_texture_file((out_basename + "_transcoded_bc7.ktx").c_str(), decoded_bc7, m_params.m_ktx2_and_basis_srgb_transfer_function); image temp(m_decoded_output_textures_unpacked_bc7[slice_index]); temp.crop(slice_desc.m_orig_width, slice_desc.m_orig_height); save_png(out_basename + "_transcoded_bc7.png", temp); } } + + if ((m_params.m_debug) && (m_decoded_output_textures_bc7[slice_index].get_pixel_width())) + { + const gpu_image& decoded_bc7 = m_decoded_output_textures_bc7[slice_index]; + + create_bc7_debug_images(slice_desc.m_orig_width, slice_desc.m_orig_height, decoded_bc7.get_ptr(), m_params.m_debug_images ? out_basename.c_str() : nullptr); + } } } // if (m_params.m_hdr) } // if (m_params.m_validate_output_data) - + return true; } - + // Make sure all the mip 0's have the same dimensions and number of mipmap levels, or we can't encode the KTX2 file. bool basis_compressor::validate_ktx2_constraints() { @@ -3227,20 +3840,80 @@ namespace basisu return true; } + + // KTX2 DFD base definitions // colorModel=KTX2_KDF_DF_MODEL_ETC1S (0xA3) // LDR ETC1S texture data in a custom format, with global codebooks - static uint8_t g_ktx2_etc1s_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA3,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x3F,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; - static uint8_t g_ktx2_etc1s_alpha_dfd[60] = { 0x3C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x38,0x0,0xA3,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x3F,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF,0x40,0x0,0x3F,0xF,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; + static uint8_t g_ktx2_etc1s_nonalpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + 0xA3,0x1,0x2,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x0,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x3F,0x0, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + static uint8_t g_ktx2_etc1s_alpha_dfd[60] = + { + 0x3C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x38,0x0, + 0xA3,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x3F,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF, + 0x40,0x0,0x3F,0xF, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; + // colorModel=KTX2_KDF_DF_MODEL_UASTC_LDR_4X4 (0xA6) // LDR UASTC 4x4 texture data in a custom block format - static uint8_t g_ktx2_uastc_ldr_4x4_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA6,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x10,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x7F,0x4,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; - static uint8_t g_ktx2_uastc_ldr_4x4_alpha_dfd[44] = { 0x2C,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x28,0x0,0xA6,0x1,0x2,0x0,0x3,0x3,0x0,0x0,0x10,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x7F,0x3,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xFF,0xFF,0xFF,0xFF }; + static uint8_t g_ktx2_uastc_ldr_4x4_nonalpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x28,0x0, + 0xA6,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x10,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x7F,0x4, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; + + static uint8_t g_ktx2_uastc_ldr_4x4_alpha_dfd[44] = + { + 0x2C,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x2,0x0,0x28,0x0, + 0xA6,0x1,0x2,0x0, + 0x3,0x3,0x0,0x0, + 0x10,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x7F,0x3, + 0x0,0x0,0x0,0x0, + 0x0,0x0,0x0,0x0, + 0xFF,0xFF,0xFF,0xFF + }; // colorModel=KTX2_KDF_DF_MODEL_UASTC_HDR_4X4 (0xA7) // Standard ASTC HDR 4x4 texture data but constrained for easy transcoding to BC6H, either highest quality or RDO optimized. - static uint8_t g_ktx2_uastc_hdr_4x4_nonalpha_dfd[44] = + static uint8_t g_ktx2_uastc_hdr_4x4_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0, // 0 totalSize 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId @@ -3249,7 +3922,7 @@ namespace basisu 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) @@ -3266,35 +3939,85 @@ namespace basisu 0x5,0x5,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) }; - // colorModel=KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE (0xA8) + // colorModel=KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE (0xA8) // Our custom intermediate format that when decoded directly outputs ASTC HDR 6x6 - static uint8_t g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd[44] = + static uint8_t g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd[44] = { 0x2C,0x0,0x0,0x0, // 0 totalSize 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber - 0xA8,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE) + 0xA8,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) 0x5,0x5,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 - 0x0,0x0,0x7F,0x80, // 7 bitLength/bitOffset/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x7F,0x80, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) 0x00, 0x00, 0x80, 0x3F // 10 sampleHigher (1.0) }; + // colorModel=KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE (0xA9) + // Custom supercompressed intermediate format, decodes directly to standard ASTC LDR 4x4-12x12. + static uint8_t g_ktx2_xuastc_ldr_intermediate_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + (uint8_t)basist::KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel (KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x7F,0x00, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.) + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + + // ASTC LDR 4x4 + static uint8_t g_ktx2_astc_ldr_dfd[44] = + { + 0x2C,0x0,0x0,0x0, // 0 totalSize + 0x0,0x0,0x0,0x0, // 1 descriptorType/vendorId + 0x2,0x0,0x28,0x0, // 2 descriptorBlockSize/versionNumber + (uint8_t)basist::KTX2_KDF_DF_MODEL_ASTC,0x1,0x1,0x0, // 3 flags, transferFunction, colorPrimaries, colorModel + 0x3,0x3,0x0,0x0, // 4 texelBlockDimension0-texelBlockDimension3 + 0x10,0x0,0x0,0x0, // 5 bytesPlane0-bytesPlane3 + 0x0,0x0,0x0,0x0, // 6 bytesPlane4-bytesPlane7 + 0x0,0x0,0x7F,0x00, // 7 bitOffset/bitLength/channelType and Qualifer flags (KHR_DF_SAMPLE_DATATYPE_FLOAT etc.), channelID=KHR_DF_CHANNEL_ASTC_DATA + 0x0,0x0,0x0,0x0, // 8 samplePosition0-samplePosition3 + 0x0,0x0,0x0,0x0, // 9 sampleLower (0.0) + 0xFF,0xFF,0xFF,0xFF // 10 sampleHigher (0xFF) + }; + bool basis_compressor::get_dfd(uint8_vec &dfd, const basist::ktx2_header &header) { - const uint8_t* pDFD; - uint32_t dfd_len; + const uint8_t* pDFD = nullptr; + uint32_t dfd_len = 0; - if (m_params.m_uastc) + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr(m_fmt_mode); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr(m_fmt_mode); + + // TODO: This was writen before m_fmt_mode existed, refactor to use that exclusively instead. + + if (is_xuastc_ldr) + { + // XUASTC LDR 4x4-12x12 + pDFD = g_ktx2_xuastc_ldr_intermediate_dfd; + dfd_len = sizeof(g_ktx2_xuastc_ldr_intermediate_dfd); + } + else if (is_astc_ldr) + { + // ASTC LDR 4x4-12x12 + pDFD = g_ktx2_astc_ldr_dfd; + dfd_len = sizeof(g_ktx2_astc_ldr_dfd); + } + else if (m_params.m_uastc) { if (m_params.m_hdr) { @@ -3302,20 +4025,26 @@ namespace basisu { case hdr_modes::cUASTC_HDR_4X4: { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_4x4); + pDFD = g_ktx2_uastc_hdr_4x4_nonalpha_dfd; dfd_len = sizeof(g_ktx2_uastc_hdr_4x4_nonalpha_dfd); break; } case hdr_modes::cASTC_HDR_6X6: { + assert(m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6); + pDFD = g_ktx2_astc_hdr_6x6_nonalpha_dfd; dfd_len = sizeof(g_ktx2_astc_hdr_6x6_nonalpha_dfd); break; } - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: + case hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE: { - pDFD = g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd; - dfd_len = sizeof(g_ktx2_astc_hdr_6x6_intermediate_nonalpha_dfd); + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); + + pDFD = g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd; + dfd_len = sizeof(g_ktx2_uastc_hdr_6x6_intermediate_nonalpha_dfd); break; } default: @@ -3328,11 +4057,15 @@ namespace basisu // Must be LDR UASTC 4x4 else if (m_any_source_image_has_alpha) { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4); + pDFD = g_ktx2_uastc_ldr_4x4_alpha_dfd; dfd_len = sizeof(g_ktx2_uastc_ldr_4x4_alpha_dfd); } else { + assert(m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4); + pDFD = g_ktx2_uastc_ldr_4x4_nonalpha_dfd; dfd_len = sizeof(g_ktx2_uastc_ldr_4x4_nonalpha_dfd); } @@ -3341,6 +4074,7 @@ namespace basisu { // Must be ETC1S. assert(!m_params.m_hdr); + assert(m_fmt_mode == basist::basis_tex_format::cETC1S); if (m_any_source_image_has_alpha) { @@ -3353,32 +4087,40 @@ namespace basisu dfd_len = sizeof(g_ktx2_etc1s_nonalpha_dfd); } } - + assert(dfd_len >= 44); dfd.resize(dfd_len); memcpy(dfd.data(), pDFD, dfd_len); + // Now modify the DFD DWORD's directly uint32_t dfd_bits = basisu::read_le_dword(dfd.data() + 3 * sizeof(uint32_t)); - // Color primaries - if ((m_params.m_hdr) && (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut)) + // Color primaries - TODO: Move this option outside of the m_astc_hdr_6x6_options struct. + //if ((m_params.m_hdr) && (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut)) + if (m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut) { dfd_bits &= ~(0xFF << 8); dfd_bits |= (basist::KTX2_DF_PRIMARIES_BT2020 << 8); } - - // Transfer function + + // Write the transfer function (linear vs. sRGB) - crucial so any decoders/transcoders know which ASTC decoding profile was used during encoding. dfd_bits &= ~(0xFF << 16); if (m_params.m_hdr) { - // TODO: In HDR mode, always write linear for now. + if (m_params.m_ktx2_and_basis_srgb_transfer_function) + { + debug_printf("WARNING: In HDR mode but m_ktx2_and_basis_srgb_transfer_function was set to true, which is being ignored while writing the KTX2 DFD transfer function field\n"); + } + + // TODO: In HDR mode, always write linear, as a sRGB transfer function doesn't make sense for HDR. dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_LINEAR << 16); } else { - if (m_params.m_ktx2_srgb_transfer_func) + // set the KTX2 DFD transfer function + if (m_params.m_ktx2_and_basis_srgb_transfer_function) dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_SRGB << 16); else dfd_bits |= (basist::KTX2_KHR_DF_TRANSFER_LINEAR << 16); @@ -3386,6 +4128,7 @@ namespace basisu basisu::write_le_dword(dfd.data() + 3 * sizeof(uint32_t), dfd_bits); + // If supercompressed, manipulate the plane bits to match the khronos ktx2 tool's output if (header.m_supercompression_scheme != basist::KTX2_SS_NONE) { uint32_t plane_bits = basisu::read_le_dword(dfd.data() + 5 * sizeof(uint32_t)); @@ -3401,16 +4144,34 @@ namespace basisu if (m_params.m_uastc) { dfd_chan0 &= ~(0xF << 24); - - // TODO: Allow the caller to override this - if (m_any_source_image_has_alpha) + + // TODO: Allow the caller to override this. Derive from swizzle? + // Only do this for UASTC LDR 4x4 or XUASTC LDR 4x4-12x12 - and now also ASTC LDR 4x4-12x12, which isn't quite standard, but we need some way of determining if the ASTC data has alpha by examining the KTX2 DFD. + if ((m_any_source_image_has_alpha) && + ((m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) || basist::basis_tex_format_is_xuastc_ldr(m_fmt_mode) || basis_tex_format_is_astc_ldr(m_fmt_mode))) + { dfd_chan0 |= (basist::KTX2_DF_CHANNEL_UASTC_RGBA << 24); + } else + { + // basist::KTX2_DF_CHANNEL_UASTC_RGB==0 dfd_chan0 |= (basist::KTX2_DF_CHANNEL_UASTC_RGB << 24); + } } basisu::write_le_dword(dfd.data() + 7 * sizeof(uint32_t), dfd_chan0); + if ((is_xuastc_ldr) || (is_astc_ldr)) + { + // Write XUASTC/ASTC LDR block dimensions + uint32_t texelBlockDimensions = basisu::read_le_dword(dfd.data() + 4 * sizeof(uint32_t)); + + texelBlockDimensions &= ~0xFFFF; + texelBlockDimensions |= ((m_fmt_mode_block_width - 1) | ((m_fmt_mode_block_height - 1) << 8)); + + basisu::write_le_dword(dfd.data() + 4 * sizeof(uint32_t), texelBlockDimensions); + } + return true; } @@ -3418,6 +4179,8 @@ namespace basisu { //bool needs_global_data = false; bool can_use_zstd = false; + bool is_xuastc_ldr = false; + bool is_astc_ldr = false; switch (m_fmt_mode) { @@ -3426,7 +4189,7 @@ namespace basisu //needs_global_data = true; break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { can_use_zstd = true; break; @@ -3441,11 +4204,50 @@ namespace basisu can_use_zstd = true; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { //needs_global_data = true; break; } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + { + // has built-in compression, no need for Zstd + is_xuastc_ldr = true; + break; + } + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + // plain ASTC LDR 4x4-12x12 - can use Zstd + is_astc_ldr = true; + can_use_zstd = true; + break; + } default: assert(0); //fmt_debug_printf("HERE 1\n"); @@ -3456,7 +4258,6 @@ namespace basisu { if ((m_params.m_ktx2_uastc_supercompression != basist::KTX2_SS_NONE) && (m_params.m_ktx2_uastc_supercompression != basist::KTX2_SS_ZSTANDARD)) { - //fmt_debug_printf("HERE 2\n"); return false; } } @@ -3507,17 +4308,52 @@ namespace basisu header.m_vk_format = basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK; else { - assert(m_params.m_hdr_mode == hdr_modes::cASTC_HDR_6X6_INTERMEDIATE); + assert(m_params.m_hdr_mode == hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE); header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; } } else { - // Either ETC1S or UASTC LDR 4x4. - assert((m_fmt_mode == basist::basis_tex_format::cETC1S) || (m_fmt_mode == basist::basis_tex_format::cUASTC4x4)); + // Either ETC1S, UASTC LDR 4x4, or XUASTC/ASTC LDR 4x4-12x12. + assert((m_fmt_mode == basist::basis_tex_format::cETC1S) || (m_fmt_mode == basist::basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr); - header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; + if (is_astc_ldr) + { + // Get the correct Vulkan format (UNORM or sRGB). + uint32_t fmt = 0; + + assert((basist::KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK + 1) == basist::KTX2_FORMAT_ASTC_4x4_SRGB_BLOCK); + + switch (m_fmt_mode) + { + case basist::basis_tex_format::cASTC_LDR_4x4: fmt = basist::KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_5x4: fmt = basist::KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_5x5: fmt = basist::KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_6x5: fmt = basist::KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_6x6: fmt = basist::KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x5: fmt = basist::KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x6: fmt = basist::KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x5: fmt = basist::KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x6: fmt = basist::KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_8x8: fmt = basist::KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x8: fmt = basist::KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_10x10: fmt = basist::KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_12x10: fmt = basist::KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK; break; + case basist::basis_tex_format::cASTC_LDR_12x12: fmt = basist::KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK; break; + default: + assert(0); + return false; + } + assert(fmt); + + header.m_vk_format = fmt + (m_params.m_ktx2_and_basis_srgb_transfer_function ? 1 : 0); + } + else + { + // A supercompressed format, i.e. not a standard format. + header.m_vk_format = basist::KTX2_VK_FORMAT_UNDEFINED; + } } header.m_type_size = 1; @@ -3605,7 +4441,7 @@ namespace basisu uint8_vec ktx2_global_data; - // Create ETC1S global supercompressed data + // Create global supercompressed data if (m_fmt_mode == basist::basis_tex_format::cETC1S) { basist::ktx2_etc1s_global_data_header etc1s_global_data_header; @@ -3659,10 +4495,11 @@ namespace basisu header.m_supercompression_scheme = basist::KTX2_SS_BASISLZ; } - else if (m_fmt_mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if ((m_fmt_mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) || (is_xuastc_ldr)) { - basisu::vector image_descs(total_levels * total_layers * total_faces); - memset((void *)image_descs.data(), 0, image_descs.size_in_bytes()); + // The global data for UASTC HDR 6x6 INTERMEDIATE and XUASTC LDR is an array of ktx2_slice_offset_len_desc's, which the transcoder needs to locate the variable length compressed slice data. + basisu::vector slice_offset_len_descs(total_levels * total_layers * total_faces); + memset((void *)slice_offset_len_descs.data(), 0, slice_offset_len_descs.size_in_bytes()); for (uint32_t slice_index = 0; slice_index < m_slice_descs.size(); slice_index++) { @@ -3680,19 +4517,19 @@ namespace basisu const uint32_t output_image_index = level_index * (total_layers * total_faces) + layer_index * total_faces + face_index; - image_descs[output_image_index].m_rgb_slice_byte_length = m_uastc_backend_output.m_slice_image_data[slice_index].size(); - image_descs[output_image_index].m_rgb_slice_byte_offset = slice_level_offsets[slice_index]; + slice_offset_len_descs[output_image_index].m_slice_byte_length = m_uastc_backend_output.m_slice_image_data[slice_index].size(); + slice_offset_len_descs[output_image_index].m_slice_byte_offset = slice_level_offsets[slice_index]; } // slice_index - append_vector(ktx2_global_data, (const uint8_t*)image_descs.data(), image_descs.size_in_bytes()); + append_vector(ktx2_global_data, (const uint8_t*)slice_offset_len_descs.data(), slice_offset_len_descs.size_in_bytes()); header.m_supercompression_scheme = basist::KTX2_SS_BASISLZ; } - + // Key values basist::ktx2_transcoder::key_value_vec key_values(m_params.m_ktx2_key_values); - + basist::ktx2_add_key_value(key_values, "KTXwriter", fmt_string("Basis Universal {}", BASISU_LIB_VERSION_STRING)); if (m_params.m_hdr) @@ -3713,11 +4550,10 @@ namespace basisu uint8_vec key_value_data; - // DFD + // DFD (Data Format Descriptor) uint8_vec dfd; if (!get_dfd(dfd, header)) { - //fmt_debug_printf("HERE 7\n"); return false; } @@ -3729,20 +4565,17 @@ namespace basisu { if (key_values[i].m_key.size() < 2) { - //fmt_debug_printf("HERE 8\n"); return false; } if (key_values[i].m_key.back() != 0) { - //fmt_debug_printf("HERE 9\n"); return false; } const uint64_t total_len = (uint64_t)key_values[i].m_key.size() + (uint64_t)key_values[i].m_value.size(); if (total_len >= UINT32_MAX) { - //fmt_debug_printf("HERE 10\n"); return false; } @@ -3764,7 +4597,7 @@ namespace basisu #if BASISU_DISABLE_KTX2_ALIGNMENT_WORKAROUND break; #endif - + // Hack to ensure the KVD block ends on a 16 byte boundary, because we have no other official way of aligning the data. uint32_t kvd_end_file_offset = kvd_file_offset + (uint32_t)key_value_data.size(); uint32_t bytes_needed_to_pad = (16 - (kvd_end_file_offset & 15)) & 15; @@ -3777,22 +4610,21 @@ namespace basisu assert(!pass); if (pass) { - //fmt_debug_printf("HERE 11\n"); return false; } if (bytes_needed_to_pad < 6) bytes_needed_to_pad += 16; - // Just add the padding. It's likely not necessary anymore, but can't really hurt. + // Just add the padding. It's likely not necessary anymore, but can't really hurt other than a tiny increase in file size. //printf("WARNING: Due to a KTX2 validator bug related to mipPadding, we must insert a dummy key into the KTX2 file of %u bytes\n", bytes_needed_to_pad); - - // We're not good - need to add a dummy key large enough to force file alignment so the mip level array gets aligned. + + // We're not good - need to add a dummy key large enough to force file alignment so the mip level array gets aligned. // We can't just add some bytes before the mip level array because ktx2check will see that as extra data in the file that shouldn't be there in ktxValidator::validateDataSize(). key_values.enlarge(1); for (uint32_t i = 0; i < (bytes_needed_to_pad - 4 - 1 - 1); i++) key_values.back().m_key.push_back(127); - + key_values.back().m_key.push_back(0); key_values.back().m_value.push_back(0); @@ -3800,13 +4632,13 @@ namespace basisu key_values.sort(); key_value_data.resize(0); - + // Try again } basisu::vector level_index_array(total_levels); memset((void *)level_index_array.data(), 0, level_index_array.size_in_bytes()); - + m_output_ktx2_file.clear(); m_output_ktx2_file.reserve(m_output_basis_file.size()); @@ -3815,8 +4647,8 @@ namespace basisu // Level index array append_vector(m_output_ktx2_file, (const uint8_t*)level_index_array.data(), level_index_array.size_in_bytes()); - - // DFD + + // Write DFD const uint8_t* pDFD = dfd.data(); uint32_t dfd_len = (uint32_t)dfd.size(); @@ -3824,7 +4656,7 @@ namespace basisu header.m_dfd_byte_length = dfd_len; append_vector(m_output_ktx2_file, pDFD, dfd_len); - // Key value data + // Write Key value data if (key_value_data.size()) { assert(kvd_file_offset == m_output_ktx2_file.size()); @@ -3834,7 +4666,7 @@ namespace basisu append_vector(m_output_ktx2_file, key_value_data); } - // Global Supercompressed Data + // Write Global Supercompressed Data if (ktx2_global_data.size()) { uint32_t ofs = m_output_ktx2_file.size() & 7; @@ -3848,14 +4680,13 @@ namespace basisu append_vector(m_output_ktx2_file, ktx2_global_data); } - // mipPadding + // Write mipPadding if (header.m_supercompression_scheme == basist::KTX2_SS_NONE) { - // We currently can't do this or the validator will incorrectly give an error. uint32_t ofs = m_output_ktx2_file.size() & 15; uint32_t padding = (16 - ofs) & 15; - // Make sure we're always aligned here (due to a validator bug). + // Make sure we're always aligned here (due to an old validator bug, which has been fixed). if (padding) { printf("Warning: KTX2 mip level data is not 16-byte aligned. This may trigger a ktx2check validation bug. Writing %u bytes of mipPadding.\n", padding); @@ -3869,8 +4700,7 @@ namespace basisu for (int level = total_levels - 1; level >= 0; level--) { level_index_array[level].m_byte_length = compressed_level_data_bytes[level].size(); - - //if (m_params.m_uastc) + if (can_use_zstd) { level_index_array[level].m_uncompressed_byte_length = level_data_bytes[level].size(); @@ -3879,7 +4709,7 @@ namespace basisu level_index_array[level].m_byte_offset = m_output_ktx2_file.size(); append_vector(m_output_ktx2_file, compressed_level_data_bytes[level]); } - + // Write final header memcpy(m_output_ktx2_file.data(), &header, sizeof(header)); @@ -3894,7 +4724,10 @@ namespace basisu total_orig_pixels += slice_desc.m_orig_width * slice_desc.m_orig_height; } - debug_printf("Total .ktx2 output file size: %u, %3.3f bits/texel\n", m_output_ktx2_file.size(), ((float)m_output_ktx2_file.size() * 8.0f) / total_orig_pixels); + m_ktx2_file_size = m_output_ktx2_file.size(); + m_ktx2_bits_per_texel = total_orig_pixels ? (m_ktx2_file_size * 8.0f) / total_orig_pixels : 0; + + fmt_debug_printf("Total .ktx2 output file size: {}, {3.3} bits/texel\n", m_ktx2_file_size, m_ktx2_bits_per_texel); return true; } @@ -3921,7 +4754,7 @@ namespace basisu std::atomic result; result.store(true); - + std::atomic opencl_failed; opencl_failed.store(false); @@ -3936,19 +4769,19 @@ namespace basisu tm.start(); basis_compressor c; - + // Dummy job pool job_pool task_jpool(1); params.m_pJob_pool = &task_jpool; // TODO: Remove this flag entirely - params.m_multithreading = true; - + params.m_multithreading = true; + // Stop using OpenCL if a failure ever occurs. if (opencl_failed) params.m_use_opencl = false; bool status = c.init(params); - + if (c.get_opencl_failed()) opencl_failed.store(true); @@ -3977,7 +4810,7 @@ namespace basisu else { results.m_error_code = basis_compressor::cECFailedInitializing; - + result = false; } @@ -3994,32 +4827,58 @@ namespace basisu return result; } - static void* basis_compress( + void* basis_compress_internal( basist::basis_tex_format mode, - const basisu::vector *pSource_images, - const basisu::vector *pSource_images_hdr, - uint32_t flags_and_quality, float uastc_rdo_quality, + const basisu::vector* pSource_images, + const basisu::vector* pSource_images_hdr, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, - image_stats* pStats) + image_stats* pStats, + int quality_level, int effort_level) { assert((pSource_images != nullptr) || (pSource_images_hdr != nullptr)); assert(!((pSource_images != nullptr) && (pSource_images_hdr != nullptr))); + if ((quality_level != -1) && (uastc_rdo_or_dct_quality != 0.0f)) + { + fmt_debug_printf("basis_compress_internal: quality_level is not -1, but uastc_rdo_or_dct_quality isn't 0!\n"); + + // Can't use both old and new-style quality control methods + uastc_rdo_or_dct_quality = 0.0f; + } + + if (!pSize) + { + error_printf("basis_compress: Need pSize parameter!\n"); + assert(0); + return nullptr; + } + + // Can't provide both LDR and HDR images + if ( ((pSource_images) && (pSource_images->size() != 0)) && + ((pSource_images_hdr) && (pSource_images_hdr->size() != 0)) + ) + { + error_printf("basis_compress: Can't provide both LDR and HDR source images!\n"); + assert(0); + return nullptr; + } + // Check input parameters if (pSource_images) { - if ((!pSource_images->size()) || (!pSize)) + if (!pSource_images->size()) { - error_printf("basis_compress: Invalid parameter\n"); + error_printf("basis_compress: No source LDR images\n"); assert(0); return nullptr; } } else { - if ((!pSource_images_hdr->size()) || (!pSize)) + if (!pSource_images_hdr->size()) { - error_printf("basis_compress: Invalid parameter\n"); + error_printf("basis_compress: No source HDR images\n"); assert(0); return nullptr; } @@ -4030,100 +4889,196 @@ namespace basisu // Initialize a job pool uint32_t num_threads = 1; if (flags_and_quality & cFlagThreaded) - num_threads = basisu::maximum(1, std::thread::hardware_concurrency()); + num_threads = basisu::maximum(1, get_num_hardware_threads()); job_pool jp(num_threads); // Initialize the compressor parameter struct basis_compressor_params comp_params; + + // Set the codec (basist::basis_tex_format) we'll be using. comp_params.set_format_mode(mode); comp_params.m_pJob_pool = &jp; comp_params.m_y_flip = (flags_and_quality & cFlagYFlip) != 0; + + // Set debug related parameters comp_params.m_debug = (flags_and_quality & cFlagDebug) != 0; comp_params.m_debug_images = (flags_and_quality & cFlagDebugImages) != 0; - // Copy the largest mipmap level - if (pSource_images) + // Set texture type: 2D, 2D array, cubemap array etc. + comp_params.m_tex_type = (basist::basis_texture_type)((flags_and_quality >> cFlagTextureTypeShift) & cFlagTextureTypeMask); + + if (comp_params.m_tex_type != basist::basis_texture_type::cBASISTexType2D) { - comp_params.m_source_images.resize(1); - comp_params.m_source_images[0] = (*pSource_images)[0]; - - // Copy the smaller mipmap levels, if any - if (pSource_images->size() > 1) + // 2D array, cubemap array, or texture video. Assume any extra images the user has supplied are actually cubemap faces, o4r array layers, or texture video frames. + // We assume the dimensions are correct here and let the compressor validate them. + // TODO: This simplified API doesn't allow the user to also specify the mipmap levels here. + if (pSource_images) { - comp_params.m_source_mipmap_images.resize(1); - comp_params.m_source_mipmap_images[0].resize(pSource_images->size() - 1); - - for (uint32_t i = 1; i < pSource_images->size(); i++) - comp_params.m_source_mipmap_images[0][i - 1] = (*pSource_images)[i]; + for (uint32_t i = 0; i < pSource_images->size(); i++) + comp_params.m_source_images.push_back((*pSource_images)[i]); + } + else + { + for (uint32_t i = 0; i < pSource_images_hdr->size(); i++) + comp_params.m_source_images_hdr.push_back((*pSource_images_hdr)[i]); } } else { - comp_params.m_source_images_hdr.resize(1); - comp_params.m_source_images_hdr[0] = (*pSource_images_hdr)[0]; - - // Copy the smaller mipmap levels, if any - if (pSource_images_hdr->size() > 1) + // Plain 2D mode. Assume any extra images the user has supplied are precomputed mipmap levels of the correct dimensions. + // Copy the largest mipmap level and mipmaps. We assume the dimensions are correct here and let the compressor validate them. + if (pSource_images) { - comp_params.m_source_mipmap_images_hdr.resize(1); - comp_params.m_source_mipmap_images_hdr[0].resize(pSource_images_hdr->size() - 1); + comp_params.m_source_images.resize(1); + comp_params.m_source_images[0] = (*pSource_images)[0]; - for (uint32_t i = 1; i < pSource_images->size(); i++) - comp_params.m_source_mipmap_images_hdr[0][i - 1] = (*pSource_images_hdr)[i]; + // Copy the smaller mipmap levels, if any + if (pSource_images->size() > 1) + { + comp_params.m_source_mipmap_images.resize(1); + comp_params.m_source_mipmap_images[0].resize(pSource_images->size() - 1); + + for (uint32_t i = 1; i < pSource_images->size(); i++) + comp_params.m_source_mipmap_images[0][i - 1] = (*pSource_images)[i]; + } + } + else + { + comp_params.m_source_images_hdr.resize(1); + comp_params.m_source_images_hdr[0] = (*pSource_images_hdr)[0]; + + // Copy the smaller mipmap levels, if any + if (pSource_images_hdr->size() > 1) + { + comp_params.m_source_mipmap_images_hdr.resize(1); + comp_params.m_source_mipmap_images_hdr[0].resize(pSource_images_hdr->size() - 1); + + for (uint32_t i = 1; i < pSource_images->size(); i++) + comp_params.m_source_mipmap_images_hdr[0][i - 1] = (*pSource_images_hdr)[i]; + } } } - + comp_params.m_multithreading = (flags_and_quality & cFlagThreaded) != 0; comp_params.m_use_opencl = (flags_and_quality & cFlagUseOpenCL) != 0; comp_params.m_write_output_basis_or_ktx2_files = false; - comp_params.m_perceptual = (flags_and_quality & cFlagSRGB) != 0; - comp_params.m_mip_srgb = comp_params.m_perceptual; + // sRGB handling - set parameters consistently + // sRGB here controls the error metrics, KTX2/.basis transfer function fields, and mipmap filtering + const bool srgb_flag = (flags_and_quality & cFlagSRGB) != 0; + + // Use sRGB colorspace metrics, channel weights + comp_params.m_perceptual = srgb_flag; + + // This will be written to the KTX2 DFD, .basis file header, also controls the ASTC profile decoding mode for ASTC LDR 4x4 - 12x12 and XUASTC LDR 4x4 - 12x12. + comp_params.m_ktx2_and_basis_srgb_transfer_function = srgb_flag; + + // Correct for sRGB transfer function during mipmapping + comp_params.m_mip_srgb = srgb_flag; + comp_params.m_mip_gen = (flags_and_quality & (cFlagGenMipsWrap | cFlagGenMipsClamp)) != 0; comp_params.m_mip_wrapping = (flags_and_quality & cFlagGenMipsWrap) != 0; - if (mode == basist::basis_tex_format::cUASTC4x4) + if (mode == basist::basis_tex_format::cUASTC_LDR_4x4) { + // Set pack level from flags comp_params.m_pack_uastc_ldr_4x4_flags = flags_and_quality & cPackUASTCLevelMask; - comp_params.m_rdo_uastc_ldr_4x4 = (flags_and_quality & cFlagUASTCRDO) != 0; - comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = uastc_rdo_quality; + + // Now optionally enable UASTC LDR 4x4 RDO. + // We used to look at the (flags_and_quality & cFlagUASTCRDO) != 0; flag to determine if we'll be using RDO here. + // The flag isn't necessary, we'll now just examine uastc_rdo_or_dct_quality and decide to enable it. + if (uastc_rdo_or_dct_quality > 0.0f) + { + comp_params.m_rdo_uastc_ldr_4x4 = true; + comp_params.m_rdo_uastc_ldr_4x4_quality_scalar = uastc_rdo_or_dct_quality; + } } else if (mode == basist::basis_tex_format::cETC1S) { - comp_params.m_etc1s_quality_level = basisu::maximum(1, flags_and_quality & 255); + // Set ETC1S quality level (codebook sizes) from flags. + comp_params.m_quality_level = basisu::maximum(1, flags_and_quality & 255); } + else if (basist::basis_tex_format_is_xuastc_ldr(mode) || basist::basis_tex_format_is_astc_ldr(mode)) + { + // Set ASTC LDR/UASTC LDR 4x4-12x12 effort level + comp_params.m_xuastc_ldr_effort_level = flags_and_quality & 255; + + // Optionally enable weight grid DCT for XUASTC. + // Valid XUASTC LDR weight grid DCT quality levels are 1-100. + if (basist::basis_tex_format_is_xuastc_ldr(mode) && (uastc_rdo_or_dct_quality != 0.0f)) + { + if ((uastc_rdo_or_dct_quality >= (float)BASISU_XUASTC_QUALITY_MIN) && (uastc_rdo_or_dct_quality < (float)BASISU_XUASTC_QUALITY_MAX)) + { + // Enable weight grid DCT usage, set quality level. + comp_params.m_xuastc_ldr_use_dct = true; + comp_params.m_quality_level = (int)uastc_rdo_or_dct_quality; + + // Also enable bounded lossy distortion mode in the normally lossless supercompressor for extra savings. + comp_params.m_xuastc_ldr_use_lossy_supercompression = true; + } + else + { + // Invalid quality level + assert(0); + return nullptr; + } + } + if (basist::basis_tex_format_is_xuastc_ldr(mode)) + { + // Set XUASTC LDR syntax + comp_params.m_xuastc_ldr_syntax = (flags_and_quality >> cFlagXUASTCLDRSyntaxShift) & cFlagXUASTCLDRSyntaxMask; + if (comp_params.m_xuastc_ldr_syntax >= (int)basist::astc_ldr_t::xuastc_ldr_syntax::cTotal) + { + error_printf("basis_compress: basis_compressor::init() failed - invalid XUASTC LDR syntax\n"); + return nullptr; + } + } + } + comp_params.m_create_ktx2_file = (flags_and_quality & cFlagKTX2) != 0; - + if (comp_params.m_create_ktx2_file) { // Set KTX2 specific parameters. if ((flags_and_quality & cFlagKTX2UASTCSuperCompression) && (comp_params.m_uastc)) comp_params.m_ktx2_uastc_supercompression = basist::KTX2_SS_ZSTANDARD; - - comp_params.m_ktx2_srgb_transfer_func = comp_params.m_perceptual; } - + comp_params.m_compute_stats = (pStats != nullptr); comp_params.m_print_stats = (flags_and_quality & cFlagPrintStats) != 0; comp_params.m_status_output = (flags_and_quality & cFlagPrintStatus) != 0; if (mode == basist::basis_tex_format::cUASTC_HDR_4x4) { + // Set UASTC HDR 4x4 effort level comp_params.m_uastc_hdr_4x4_options.set_quality_level(flags_and_quality & cPackUASTCLevelMask); } - else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + else if ((mode == basist::basis_tex_format::cASTC_HDR_6x6) || (mode == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) { + // Set ASTC HDR 6x6/UASTC HDR 6x6 effort level comp_params.m_astc_hdr_6x6_options.set_user_level(flags_and_quality & cPackUASTCLevelMask); - comp_params.m_astc_hdr_6x6_options.m_lambda = uastc_rdo_quality; - comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = (flags_and_quality & cFlagREC2020) != 0; + + // Set lambda (rate-distortion tradeoff) + comp_params.m_astc_hdr_6x6_options.m_lambda = uastc_rdo_or_dct_quality; } + // TODO: REC2020 isn't specific to HDR 6x6 anymore, it's always used for KTX2 files. + // This will be written to the KTX2 DFD. + comp_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = (flags_and_quality & cFlagREC2020) != 0; + comp_params.m_validate_output_data = (flags_and_quality & cFlagValidateOutput) != 0; + + // Now set the unified quality/effort level, if they've specified it. + // This will override some of the lower-level options set above, or leave them alone if -1. + if ((quality_level != -1) || (effort_level != -1)) + { + comp_params.set_format_mode_and_quality_effort(mode, quality_level, effort_level, false); + } // Create the compressor, initialize it, and process the input basis_compressor comp; @@ -4156,6 +5111,7 @@ namespace basisu error_printf("basis_compress: Out of memory\n"); return nullptr; } + memcpy(pFile_data, pFile_data_vec->get_ptr(), pFile_data_vec->size()); *pSize = pFile_data_vec->size(); @@ -4171,27 +5127,47 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats) { - return basis_compress(mode, &source_images, nullptr, flags_and_quality, uastc_rdo_quality, pSize, pStats); + return basis_compress_internal(mode, &source_images, nullptr, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats, -1, -1); + } + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats) + { + return basis_compress_internal(mode, &source_images, nullptr, flags_and_quality, 0.0f, pSize, pStats, quality_level, effort_level); } void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images_hdr, - uint32_t flags_and_quality, float lambda, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats) { - return basis_compress(mode, nullptr, &source_images_hdr, flags_and_quality, lambda, pSize, pStats); + return basis_compress_internal(mode, nullptr, &source_images_hdr, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats, -1, -1); + } + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images_hdr, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats) + { + return basis_compress_internal(mode, nullptr, &source_images_hdr, flags_and_quality, 0.0f, pSize, pStats, quality_level, effort_level); } void* basis_compress( basist::basis_tex_format mode, const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats) { @@ -4219,9 +5195,43 @@ namespace basisu for (uint32_t y = 0; y < height; y++) memcpy(source_image[0].get_ptr() + y * width, (const color_rgba*)pImageRGBA + y * pitch_in_pixels, width * sizeof(color_rgba)); - return basis_compress(mode, source_image, flags_and_quality, uastc_rdo_quality, pSize, pStats); + return basis_compress(mode, source_image, flags_and_quality, uastc_rdo_or_dct_quality, pSize, pStats); } + void* basis_compress2( + basist::basis_tex_format mode, + const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats) + { + if (!pitch_in_pixels) + pitch_in_pixels = width; + + if ((!pImageRGBA) || (!width) || (!height) || (pitch_in_pixels < width) || (!pSize)) + { + error_printf("basis_compress: Invalid parameter\n"); + assert(0); + return nullptr; + } + + *pSize = 0; + + if ((width > BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION) || (height > BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION)) + { + error_printf("basis_compress: Image too large\n"); + return nullptr; + } + + // Copy the source image + basisu::vector source_image(1); + source_image[0].crop(width, height, width, g_black_color, false); + for (uint32_t y = 0; y < height; y++) + memcpy(source_image[0].get_ptr() + y * width, (const color_rgba*)pImageRGBA + y * pitch_in_pixels, width * sizeof(color_rgba)); + + return basis_compress2(mode, source_image, flags_and_quality, quality_level, effort_level, pSize, pStats); + } + void basis_free_data(void* p) { free(p); @@ -4241,7 +5251,7 @@ namespace basisu const uint32_t W = 1024, H = 1024; basisu::vector images; image& img = images.enlarge(1)->resize(W, H); - + const uint32_t NUM_RAND_LETTERS = 6000;// 40000; rand r; @@ -4283,7 +5293,7 @@ namespace basisu error_printf("basis_benchmark_etc1s_opencl: basis_compress() failed (CPU)!\n"); return false; } - + best_cpu_time = minimum(best_cpu_time, cpu_time); basis_free_data(pComp_data); @@ -4326,8 +5336,11 @@ namespace basisu } printf("Best GPU time: %3.3f\n", best_gpu_time); - + return best_gpu_time < best_cpu_time; } } // namespace basisu + + + diff --git a/encoder/basisu_comp.h b/encoder/basisu_comp.h index e761eac..f2113dc 100644 --- a/encoder/basisu_comp.h +++ b/encoder/basisu_comp.h @@ -1,5 +1,5 @@ // basisu_comp.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,9 +20,10 @@ #include "basisu_uastc_enc.h" #include "basisu_uastc_hdr_4x4_enc.h" #include "basisu_astc_hdr_6x6_enc.h" +#include "basisu_astc_ldr_encode.h" -#define BASISU_LIB_VERSION 160 -#define BASISU_LIB_VERSION_STRING "1.60" +#define BASISU_LIB_VERSION 200 +#define BASISU_LIB_VERSION_STRING "2.00" #ifndef BASISD_SUPPORT_KTX2 #error BASISD_SUPPORT_KTX2 is undefined @@ -43,10 +44,10 @@ namespace basisu const uint32_t BASISU_MAX_SUPPORTED_TEXTURE_DIMENSION = 16384; // Allow block's color distance to increase by 1.5 while searching for an alternative nearby endpoint. - const float BASISU_DEFAULT_ENDPOINT_RDO_THRESH = 1.5f; - + const float BASISU_DEFAULT_ENDPOINT_RDO_THRESH = 1.5f; + // Allow block's color distance to increase by 1.25 while searching the selector history buffer for a close enough match. - const float BASISU_DEFAULT_SELECTOR_RDO_THRESH = 1.25f; + const float BASISU_DEFAULT_SELECTOR_RDO_THRESH = 1.25f; const int BASISU_DEFAULT_QUALITY = 128; const float BASISU_DEFAULT_HYBRID_SEL_CB_QUALITY_THRESH = 2.0f; @@ -54,6 +55,8 @@ namespace basisu const uint32_t BASISU_MAX_IMAGE_DIMENSION = 16384; const uint32_t BASISU_QUALITY_MIN = 1; const uint32_t BASISU_QUALITY_MAX = 255; + const uint32_t BASISU_XUASTC_QUALITY_MIN = 1; + const uint32_t BASISU_XUASTC_QUALITY_MAX = 100; const uint32_t BASISU_MAX_ENDPOINT_CLUSTERS = basisu_frontend::cMaxEndpointClusters; const uint32_t BASISU_MAX_SELECTOR_CLUSTERS = basisu_frontend::cMaxSelectorClusters; @@ -75,7 +78,7 @@ namespace basisu m_filename.clear(); m_width = 0; m_height = 0; - + m_basis_rgb_avg_psnr = 0.0f; m_basis_rgb_avg_log2_psnr = 0.0f; @@ -94,7 +97,7 @@ namespace basisu m_bc7_luma_709_psnr = 0.0f; m_bc7_luma_601_psnr = 0.0f; m_bc7_luma_709_ssim = 0.0f; - + m_best_etc1s_rgb_avg_psnr = 0.0f; m_best_etc1s_luma_709_psnr = 0.0f; m_best_etc1s_luma_601_psnr = 0.0f; @@ -128,8 +131,8 @@ namespace basisu float m_bc7_luma_709_psnr; float m_bc7_luma_601_psnr; float m_bc7_luma_709_ssim; - - // LDR: Highest achievable quality ETC1S statistics + + // LDR: Highest achievable quality ETC1S statistics, for development/comparison float m_best_etc1s_rgb_avg_psnr; float m_best_etc1s_luma_709_psnr; float m_best_etc1s_luma_601_psnr; @@ -141,11 +144,11 @@ namespace basisu enum class hdr_modes { // standard but constrained ASTC HDR 4x4 tex data that can be rapidly transcoded to BC6H - cUASTC_HDR_4X4, + cUASTC_HDR_4X4, // standard RDO optimized or non-RDO (highest quality) ASTC HDR 6x6 tex data that can be rapidly re-encoded to BC6H cASTC_HDR_6X6, // a custom intermediate format based off ASTC HDR that can be rapidly decoded straight to ASTC HDR or re-encoded to BC6H - cASTC_HDR_6X6_INTERMEDIATE, + cUASTC_HDR_6X6_INTERMEDIATE, cTotal }; @@ -230,17 +233,21 @@ namespace basisu bool m_changed; }; + // Low-level direct compressor parameters. + // Also see basis_compress() below for a simplified C-style interface. struct basis_compressor_params { basis_compressor_params() : - m_compression_level((int)BASISU_DEFAULT_COMPRESSION_LEVEL, 0, (int)BASISU_MAX_COMPRESSION_LEVEL), + m_xuastc_or_astc_ldr_basis_tex_format(-1, -1, INT_MAX), + // Note the ETC1S default compression/effort level is 2, not the command line default of 1. + m_etc1s_compression_level((int)BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL, 0, (int)BASISU_MAX_ETC1S_COMPRESSION_LEVEL), m_selector_rdo_thresh(BASISU_DEFAULT_SELECTOR_RDO_THRESH, 0.0f, 1e+10f), m_endpoint_rdo_thresh(BASISU_DEFAULT_ENDPOINT_RDO_THRESH, 0.0f, 1e+10f), m_mip_scale(1.0f, .000125f, 4.0f), m_mip_smallest_dimension(1, 1, 16384), m_etc1s_max_endpoint_clusters(512), m_etc1s_max_selector_clusters(512), - m_etc1s_quality_level(-1), + m_quality_level(-1), m_pack_uastc_ldr_4x4_flags(cPackUASTCLevelDefault), m_rdo_uastc_ldr_4x4_quality_scalar(1.0f, 0.001f, 50.0f), m_rdo_uastc_ldr_4x4_dict_size(BASISU_RDO_UASTC_DICT_SIZE_DEFAULT, BASISU_RDO_UASTC_DICT_SIZE_MIN, BASISU_RDO_UASTC_DICT_SIZE_MAX), @@ -253,8 +260,14 @@ namespace basisu m_resample_factor(0.0f, .00125f, 100.0f), m_ktx2_uastc_supercompression(basist::KTX2_SS_NONE), m_ktx2_zstd_supercompression_level(6, INT_MIN, INT_MAX), + m_transcode_flags(0, 0, UINT32_MAX), m_ldr_hdr_upconversion_nit_multiplier(0.0f, 0.0f, basist::MAX_HALF_FLOAT), m_ldr_hdr_upconversion_black_bias(0.0f, 0.0f, 1.0f), + m_xuastc_ldr_effort_level(astc_ldr::EFFORT_LEVEL_DEF, astc_ldr::EFFORT_LEVEL_MIN, astc_ldr::EFFORT_LEVEL_MAX), + m_xuastc_ldr_syntax((int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd, (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith, (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd), + m_ls_min_psnr(35.0f, 0.0f, 100.0f), m_ls_min_alpha_psnr(38.0f, 0.0f, 100.0f), + m_ls_thresh_psnr(1.5f, 0.0f, 100.0f), m_ls_thresh_alpha_psnr(0.75f, 0.0f, 100.0f), + m_ls_thresh_edge_psnr(1.0f, 0.0f, 100.00f), m_ls_thresh_edge_alpha_psnr(0.5f, 0.0f, 100.00f), m_pJob_pool(nullptr) { clear(); @@ -262,9 +275,12 @@ namespace basisu void clear() { + m_format_mode = basist::basis_tex_format::cETC1S; + m_uastc.clear(); m_hdr.clear(); m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_use_opencl.clear(); m_status_output.clear(); @@ -286,7 +302,7 @@ namespace basisu m_selector_rdo_thresh.clear(); m_read_source_images.clear(); m_write_output_basis_or_ktx2_files.clear(); - m_compression_level.clear(); + m_etc1s_compression_level.clear(); m_compute_stats.clear(); m_print_stats.clear(); m_check_for_alpha.clear(); @@ -301,7 +317,7 @@ namespace basisu m_no_endpoint_rdo.clear(); m_endpoint_rdo_thresh.clear(); - + m_mip_gen.clear(); m_mip_scale.clear(); m_mip_filter = "kaiser"; @@ -315,7 +331,7 @@ namespace basisu m_etc1s_max_endpoint_clusters = 0; m_etc1s_max_selector_clusters = 0; - m_etc1s_quality_level = -1; + m_quality_level = -1; m_tex_type = basist::cBASISTexType2D; m_userdata0 = 0; @@ -342,37 +358,62 @@ namespace basisu m_ktx2_uastc_supercompression = basist::KTX2_SS_NONE; m_ktx2_key_values.clear(); m_ktx2_zstd_supercompression_level.clear(); - m_ktx2_srgb_transfer_func.clear(); + m_ktx2_and_basis_srgb_transfer_function.clear(); m_validate_output_data.clear(); + m_transcode_flags.clear(); m_ldr_hdr_upconversion_srgb_to_linear.clear(); m_hdr_favor_astc.clear(); - + m_uastc_hdr_4x4_options.init(); m_astc_hdr_6x6_options.clear(); m_ldr_hdr_upconversion_nit_multiplier.clear(); m_ldr_hdr_upconversion_black_bias.clear(); + m_xuastc_ldr_effort_level.clear(); + m_xuastc_ldr_use_dct.clear(); + m_xuastc_ldr_use_lossy_supercompression.clear(); + m_xuastc_ldr_force_disable_subsets.clear(); + m_xuastc_ldr_force_disable_rgb_dual_plane.clear(); + m_xuastc_ldr_syntax.clear(); + + m_ls_min_psnr.clear(); + m_ls_min_alpha_psnr.clear(); + m_ls_thresh_psnr.clear(); + m_ls_thresh_alpha_psnr.clear(); + m_ls_thresh_edge_psnr.clear(); + m_ls_thresh_edge_alpha_psnr.clear(); + for (uint32_t i = 0; i < 4; i++) + m_xuastc_ldr_channel_weights[i] = 1; + m_xuastc_ldr_blurring.clear(); + m_pJob_pool = nullptr; } - + // Configures the compressor's mode by setting the proper parameters (which were preserved for backwards compatibility with old code). - void set_format_mode(basist::basis_tex_format m) + // This is by far the preferred way of controlling which codec mode the compressor will select. + void set_format_mode(basist::basis_tex_format mode) { - switch (m) + m_format_mode = mode; + + switch (mode) { case basist::basis_tex_format::cETC1S: { + // ETC1S + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = false; m_uastc = false; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter break; } - case basist::basis_tex_format::cUASTC4x4: + case basist::basis_tex_format::cUASTC_LDR_4x4: { + // UASTC LDR 4x4 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = false; m_uastc = true; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter @@ -380,6 +421,8 @@ namespace basisu } case basist::basis_tex_format::cUASTC_HDR_4x4: { + // UASTC HDR 4x4 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; @@ -387,16 +430,56 @@ namespace basisu } case basist::basis_tex_format::cASTC_HDR_6x6: { + // ASTC HDR 6x6 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; m_hdr_mode = hdr_modes::cASTC_HDR_6X6; break; } - case basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: { + // UASTC HDR 6x6 + m_xuastc_or_astc_ldr_basis_tex_format = -1; m_hdr = true; m_uastc = true; - m_hdr_mode = hdr_modes::cASTC_HDR_6X6_INTERMEDIATE; + m_hdr_mode = hdr_modes::cUASTC_HDR_6X6_INTERMEDIATE; + break; + } + case basist::basis_tex_format::cXUASTC_LDR_4x4: + case basist::basis_tex_format::cXUASTC_LDR_5x4: + case basist::basis_tex_format::cXUASTC_LDR_5x5: + case basist::basis_tex_format::cXUASTC_LDR_6x5: + case basist::basis_tex_format::cXUASTC_LDR_6x6: + case basist::basis_tex_format::cXUASTC_LDR_8x5: + case basist::basis_tex_format::cXUASTC_LDR_8x6: + case basist::basis_tex_format::cXUASTC_LDR_10x5: + case basist::basis_tex_format::cXUASTC_LDR_10x6: + case basist::basis_tex_format::cXUASTC_LDR_8x8: + case basist::basis_tex_format::cXUASTC_LDR_10x8: + case basist::basis_tex_format::cXUASTC_LDR_10x10: + case basist::basis_tex_format::cXUASTC_LDR_12x10: + case basist::basis_tex_format::cXUASTC_LDR_12x12: + case basist::basis_tex_format::cASTC_LDR_4x4: + case basist::basis_tex_format::cASTC_LDR_5x4: + case basist::basis_tex_format::cASTC_LDR_5x5: + case basist::basis_tex_format::cASTC_LDR_6x5: + case basist::basis_tex_format::cASTC_LDR_6x6: + case basist::basis_tex_format::cASTC_LDR_8x5: + case basist::basis_tex_format::cASTC_LDR_8x6: + case basist::basis_tex_format::cASTC_LDR_10x5: + case basist::basis_tex_format::cASTC_LDR_10x6: + case basist::basis_tex_format::cASTC_LDR_8x8: + case basist::basis_tex_format::cASTC_LDR_10x8: + case basist::basis_tex_format::cASTC_LDR_10x10: + case basist::basis_tex_format::cASTC_LDR_12x10: + case basist::basis_tex_format::cASTC_LDR_12x12: + { + // ASTC LDR 4x4-12x12 or XUASTC LDR 4x4-12x12 + m_xuastc_or_astc_ldr_basis_tex_format = (int)mode; + m_hdr = false; + m_uastc = true; + m_hdr_mode = hdr_modes::cUASTC_HDR_4X4; // doesn't matter break; } default: @@ -405,39 +488,84 @@ namespace basisu } } - // By default we generate LDR ETC1S data. - // if m_uastc is true but m_hdr is not true, we generate UASTC 4x4 LDR data (8bpp with or without RDO). - // if m_uastc is true and m_hdr is true, we generate 4x4 or 6x6 HDR data (either standard ASTC, constrained ASTC, RDO ASTC, or intermediate), controlled by m_hdr_mode. + // Like set_format_mode() but also sets the effort and quality parameters appropriately for the selected mode. + // "Effort" (perf. vs. highest achievable quality) and "quality" (quality vs. bitrate) parameters are now mode dependent. + // Effort ranges from [0,10] and quality ranges from [1,100], unless they are -1 in which case you get the codec's default settings. + bool set_format_mode_and_effort(basist::basis_tex_format mode, int effort = -1, bool set_defaults = true); + bool set_format_mode_and_quality_effort(basist::basis_tex_format mode, int quality = -1, int effort = -1, bool set_defaults = true); + // Sets all the sRGB-related options (m_perceptual, m_mip_srgb, m_ktx2_and_basis_srgb_transfer_function) to the specified value. + void set_srgb_options(bool srgb_flag) + { + m_perceptual = srgb_flag; + m_mip_srgb = srgb_flag; + m_ktx2_and_basis_srgb_transfer_function = srgb_flag; + } + + // Simpler helpers - I wish this was easier, but backwards API compat is also valuable. + bool is_etc1s() const + { + return !m_uastc; + } + + bool is_uastc_ldr_4x4() const + { + return m_uastc && !m_hdr && (m_xuastc_or_astc_ldr_basis_tex_format == -1); + } + + bool is_uastc_hdr_4x4() const + { + return m_uastc && m_hdr && (m_hdr_mode == hdr_modes::cUASTC_HDR_4X4); + } + + // By default we generate LDR ETC1S data. + // Ideally call set_format_mode() above instead of directly manipulating the below fields. These individual parameters are for backwards API compatibility. + // - If m_uastc is false you get ETC1S (the default). + // - If m_uastc is true, and m_hdr is not true, and m_xuastc_or_astc_ldr_basis_tex_format==-1, we generate UASTC 4x4 LDR data (8bpp with or without RDO). + // - If m_uastc is true, and m_hdr is not true, and m_xuastc_or_astc_ldr_basis_tex_format!=-1, we generate XUASTC 4x4-12x12 or ASTC 4x4-12x12 LDR data. + // - If m_uastc is true and m_hdr is true, we generate 4x4 or 6x6 HDR data, controlled by m_hdr_mode. + // True to generate UASTC .basis/.KTX2 file data, otherwise ETC1S. - // Should be true for any non-ETC1S format (UASTC 4x4 LDR, UASTC 4x4 HDR, RDO ASTC 6x6 HDR, and ASTC 6x6 HDR intermediate). + // Should be true for any non-ETC1S format (UASTC 4x4 LDR, UASTC 4x4 HDR, RDO ASTC 6x6 HDR, UASTC 6x6 HDR, or ASTC/XUASTC LDR 4x4-12x12). + // Note: Ideally call set_format_mode() or set_format_mode_and_quality_effort() above instead. + // Many of these individual parameters are for backwards API compatibility. bool_param m_uastc; // Set m_hdr to true to switch to UASTC HDR mode. m_hdr_mode then controls which format is output. // m_hdr_mode then controls which format is output (4x4, 6x6, or 6x6 intermediate). + // Note: Ideally call set_format_mode() instead. This is for backwards API compatibility. bool_param m_hdr; - + // If m_hdr is true, this specifies which mode we operate in (currently UASTC 4x4 HDR or ASTC 6x6 HDR). Defaults to UASTC 4x4 HDR for backwards compatibility. + // Note: Ideally call set_format_mode() instead. This is for backwards API compatibility. hdr_modes m_hdr_mode; + // If not -1: Generate XUASTC or ASTC LDR 4x4-12x12 files in the specified basis_tex_format (which also sets the ASTC block size). If -1 (the default), don't generate XUASTC/ASTC LDR files. + // m_uastc must also be set to true if this is not -1. + // Note: Ideally call set_format_mode() instead. + param m_xuastc_or_astc_ldr_basis_tex_format; // enum basis_tex_format + + // True to enable OpenCL if it's available. The compressor will fall back to CPU encoding if something goes wrong. bool_param m_use_opencl; - // If m_read_source_images is true, m_source_filenames (and optionally m_source_alpha_filenames) contains the filenames of PNG etc. images to read. + // If m_read_source_images is true, m_source_filenames (and optionally m_source_alpha_filenames) contains the filenames of PNG etc. images to read. // Otherwise, the compressor processes the images in m_source_images or m_source_images_hdr. basisu::vector m_source_filenames; basisu::vector m_source_alpha_filenames; - + + // An array of 2D LDR/SDR source images. basisu::vector m_source_images; - + + // An array of 2D HDR source images. basisu::vector m_source_images_hdr; - + // Stores mipmaps starting from level 1. Level 0 is still stored in m_source_images, as usual. // If m_source_mipmaps isn't empty, automatic mipmap generation isn't done. m_source_mipmaps.size() MUST equal m_source_images.size() or the compressor returns an error. // The compressor applies the user-provided swizzling (in m_swizzle) to these images. basisu::vector< basisu::vector > m_source_mipmap_images; basisu::vector< basisu::vector > m_source_mipmap_images_hdr; - + // Filename of the output basis/ktx2 file std::string m_out_filename; @@ -448,20 +576,24 @@ namespace basisu // If true, the compressor will print basis status to stdout during compression. bool_param m_status_output; - + // Output debug information during compression bool_param m_debug; + + // Low-level ETC1S data validation during encoding (slower/development). bool_param m_validate_etc1s; - + // m_debug_images is pretty slow bool_param m_debug_images; - // ETC1S compression level, from 0 to BASISU_MAX_COMPRESSION_LEVEL (higher is slower). + // ETC1S compression effort level, from 0 to BASISU_MAX_ETC1S_COMPRESSION_LEVEL (higher is slower). // This parameter controls numerous internal encoding speed vs. compression efficiency/performance tradeoffs. // Note this is NOT the same as the ETC1S quality level, and most users shouldn't change this. - param m_compression_level; - - // Use perceptual sRGB colorspace metrics instead of linear + param m_etc1s_compression_level; + + // Use perceptual sRGB colorspace metrics instead of linear. + // Note: You probably also want to set m_ktx2_srgb_transfer_func to match. + // Note: This member variable was previously called "m_perceptual". bool_param m_perceptual; // Disable selector RDO, for faster compression but larger files @@ -476,47 +608,53 @@ namespace basisu // Write the output basis/ktx2 file to disk using m_out_filename bool_param m_write_output_basis_or_ktx2_files; - - // Compute and display image metrics + + // Compute and display image metrics bool_param m_compute_stats; // Print stats to stdout, if m_compute_stats is true. bool_param m_print_stats; - + // Check to see if any input image has an alpha channel, if so then the output basis/ktx2 file will have alpha channels bool_param m_check_for_alpha; - + // Always put alpha slices in the output basis/ktx2 file, even when the input doesn't have alpha - bool_param m_force_alpha; - bool_param m_multithreading; + bool_param m_force_alpha; + // True to enable multithreading in various compressors. + // Note currently, some compressors (like ASTC/XUASTC LDR) will utilize threading anyway if the job pool is more than one thread. + bool_param m_multithreading; + // Split the R channel to RGB and the G channel to alpha, then write a basis/ktx2 file with alpha channels uint8_t m_swizzle[4]; + // Renormalize normal map normals after loading image bool_param m_renormalize; // If true the front end will not use 2 level endpoint codebook searching, for slightly higher quality but much slower execution. - // Note some m_compression_level's disable this automatically. + // Note some m_etc1s_compression_level's disable this automatically. bool_param m_disable_hierarchical_endpoint_codebooks; - + // mipmap generation parameters bool_param m_mip_gen; param m_mip_scale; std::string m_mip_filter; bool_param m_mip_srgb; bool_param m_mip_premultiplied; // not currently supported - bool_param m_mip_renormalize; + bool_param m_mip_renormalize; bool_param m_mip_wrapping; bool_param m_mip_fast; param m_mip_smallest_dimension; - - // ETC1S codebook size (quality) control. + + // ETC1S codebook size (quality) control. // If m_etc1s_quality_level != -1, it controls the quality level. It ranges from [1,255] or [BASISU_QUALITY_MIN, BASISU_QUALITY_MAX]. // Otherwise m_max_endpoint_clusters/m_max_selector_clusters controls the codebook sizes directly. uint32_t m_etc1s_max_endpoint_clusters; uint32_t m_etc1s_max_selector_clusters; - int m_etc1s_quality_level; + // Quality level (bitrate vs. distortion tradeoff) control for ETC1S or XUASTC LDR 4x4-12x12 (must not be -1 for DCT to be used in XUASTC LDR 4x4 mode) + int m_quality_level; + // m_tex_type, m_userdata0, m_userdata1, m_framerate - These fields go directly into the .basis file header. basist::basis_texture_type m_tex_type; uint32_t m_userdata0; @@ -527,7 +665,7 @@ namespace basisu // cPackUASTCLevelDefault, etc. uint32_t m_pack_uastc_ldr_4x4_flags; bool_param m_rdo_uastc_ldr_4x4; - param m_rdo_uastc_ldr_4x4_quality_scalar; + param m_rdo_uastc_ldr_4x4_quality_scalar; // RDO lambda for UASTC 4x4 LDR param m_rdo_uastc_ldr_4x4_dict_size; param m_rdo_uastc_ldr_4x4_max_smooth_block_error_scale; param m_rdo_uastc_ldr_4x4_smooth_block_max_std_dev; @@ -536,10 +674,12 @@ namespace basisu bool_param m_rdo_uastc_ldr_4x4_favor_simpler_modes_in_rdo_mode; bool_param m_rdo_uastc_ldr_4x4_multithreading; + // Resample input texture after loading param m_resample_width; param m_resample_height; param m_resample_factor; + // ETC1S global codebook control const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks; // KTX2 specific parameters. @@ -548,19 +688,28 @@ namespace basisu basist::ktx2_supercompression m_ktx2_uastc_supercompression; basist::ktx2_transcoder::key_value_vec m_ktx2_key_values; param m_ktx2_zstd_supercompression_level; - bool_param m_ktx2_srgb_transfer_func; + + // Note: The default for this parameter (which used to be "m_ktx2_srgb_transfer_func") used to be false, now setting this to true and renaming to m_ktx2_and_basis_srgb_transfer_function. + // Also see m_perceptual and m_mip_srgb, which should in most uses be the same. + // This also controls the XUASTC LDR ASTC decode profile (linear vs. sRGB) in the simulated decoder block. + // For XUASTC LDR, it's also still used when generating .basis files vs. .KTX2. + bool_param m_ktx2_and_basis_srgb_transfer_function; // false = linear transfer function, true = sRGB transfer function uastc_hdr_4x4_codec_options m_uastc_hdr_4x4_options; astc_6x6_hdr::astc_hdr_6x6_global_config m_astc_hdr_6x6_options; + // True to try transcoding the generated output after compression to a few formats. bool_param m_validate_output_data; + + // The flags to use while transcoding if m_validate_output_data + param m_transcode_flags; // LDR->HDR upconversion parameters. - // - // If true, LDR images (such as PNG) will be converted to normalized [0,1] linear light (via a sRGB->Linear conversion), or absolute luminance (nits or candelas per meter squared), and then processed as HDR. + // + // If true, LDR images (such as PNG) will be converted to normalized [0,1] linear light (via a sRGB->Linear conversion), or absolute luminance (nits or candelas per meter squared), and then processed as HDR. // Otherwise, LDR images are assumed to already be in linear light (i.e. they don't use the sRGB transfer function). bool_param m_ldr_hdr_upconversion_srgb_to_linear; - + // m_ldr_hdr_upconversion_nit_multiplier is only used when loading SDR/LDR images and compressing to an HDR output format. // By default m_ldr_hdr_upconversion_nit_multiplier is 0. It's an override for the default. // When loading LDR images, a default multiplier of 1.0 will be used in UASTC 4x4 HDR mode. Partially for backwards compatibility with previous library releases, and also because it doesn't really matter with this encoder what the multiplier is. @@ -571,10 +720,35 @@ namespace basisu // Defaults to no bias (0.0f). param m_ldr_hdr_upconversion_black_bias; - // If true, ASTC HDR quality is favored more than BC6H quality. Otherwise it's a rough balance. + // If true, ASTC HDR quality is favored more than BC6H quality by the dual target encoder. Otherwise it's a rough balance. + // UASTC HDR 4x4 bool_param m_hdr_favor_astc; + // XUASTC LDR 4x4-12x12 specific options + param m_xuastc_ldr_effort_level; + bool_param m_xuastc_ldr_use_dct; // set the DCT quality above using m_quality_level, [1,100] + bool_param m_xuastc_ldr_use_lossy_supercompression; // allows the compressor to introduce a bounded amount of distortion if doing so would make smaller files (actually ASTC or XUASTC) + bool_param m_xuastc_ldr_force_disable_subsets; // disable 2-3 subset usage in all effort levels, faster encoding, faster transcoding to BC7, but lower quality) + bool_param m_xuastc_ldr_force_disable_rgb_dual_plane; // disable RGB dual plane usage (still can use dual plane on alpha blocks), for faster transcoding to BC7 but lower quality + param m_xuastc_ldr_syntax; // favor faster decompression over ratio, default is basist::astc_ldr_t::xuastc_ldr_syntax::cFullZstd (fastest transcoding but lower ratio) + uint32_t m_xuastc_ldr_channel_weights[4]; + bool_param m_xuastc_ldr_blurring; // experimental, not recommended, very slow + + // XUASTC Lossy supercompression PSNR threshold parameters + param m_ls_min_psnr, m_ls_min_alpha_psnr; + param m_ls_thresh_psnr, m_ls_thresh_alpha_psnr; + param m_ls_thresh_edge_psnr, m_ls_thresh_edge_alpha_psnr; + + // Job pool, MUST not be nullptr; job_pool *m_pJob_pool; + + // Returns the current format mode as set by set_format_mode() above. + // Because of backwards API compatibility we don't use this directly yet, it's just here to aid the transition to the new API. + basist::basis_tex_format get_format_mode() const { return m_format_mode; } + + private: + // This is set by set_format_mode() above. For backwards API compat we don't use it directly, it's just here to aid the transition to the new API. + basist::basis_tex_format m_format_mode; }; // Important: basisu_encoder_init() MUST be called first before using this class. @@ -588,7 +762,7 @@ namespace basisu // Note it *should* be possible to call init() multiple times with different inputs, but this scenario isn't well tested. Ideally, create 1 object, compress, then delete it. bool init(const basis_compressor_params ¶ms); - + enum error_code { cECSuccess = 0, @@ -602,45 +776,63 @@ namespace basisu cECFailedCreateBasisFile, cECFailedWritingOutput, cECFailedUASTCRDOPostProcess, - cECFailedCreateKTX2File + cECFailedCreateKTX2File, + cECFailedInvalidParameters }; error_code process(); // The output .basis file will always be valid of process() succeeded. const uint8_vec &get_output_basis_file() const { return m_output_basis_file; } - + // The output .ktx2 file will only be valid if m_create_ktx2_file was true and process() succeeded. const uint8_vec& get_output_ktx2_file() const { return m_output_ktx2_file; } const basisu::vector &get_stats() const { return m_stats; } - uint32_t get_basis_file_size() const { return m_basis_file_size; } + // Sum of all slice orig pixels. Intended for statistics display. + uint64_t get_total_slice_orig_texels() const { return m_total_slice_orig_texels; } + + uint64_t get_basis_file_size() const { return m_basis_file_size; } double get_basis_bits_per_texel() const { return m_basis_bits_per_texel; } + uint64_t get_ktx2_file_size() const { return m_ktx2_file_size; } + double get_ktx2_bits_per_texel() const { return m_ktx2_bits_per_texel; } + bool get_any_source_image_has_alpha() const { return m_any_source_image_has_alpha; } bool get_opencl_failed() const { return m_opencl_failed; } - + private: basis_compressor_params m_params; - + opencl_context_ptr m_pOpenCL_context; - basist::basis_tex_format m_fmt_mode; - + // the output mode/codec + basist::basis_tex_format m_fmt_mode; + + // the output mode/codec's block width/height + uint32_t m_fmt_mode_block_width; + uint32_t m_fmt_mode_block_height; + + // Note these images are expanded if necessary (duplicating cols/rows) to account for block dimensions. basisu::vector m_slice_images; basisu::vector m_slice_images_hdr; basisu::vector m_stats; - uint32_t m_basis_file_size; + uint64_t m_total_slice_orig_texels; + + uint64_t m_basis_file_size; double m_basis_bits_per_texel; + uint64_t m_ktx2_file_size; + double m_ktx2_bits_per_texel; + basisu_backend_slice_desc_vec m_slice_descs; uint32_t m_total_blocks; - + basisu_frontend m_frontend; // These are 4x4 blocks. @@ -658,7 +850,7 @@ namespace basisu basisu::vector m_decoded_output_textures; // BC6H in HDR mode basisu::vector m_decoded_output_textures_unpacked; - + basisu::vector m_decoded_output_textures_bc7; basisu::vector m_decoded_output_textures_unpacked_bc7; @@ -669,16 +861,16 @@ namespace basisu uint8_vec m_output_basis_file; uint8_vec m_output_ktx2_file; - + basisu::vector m_uastc_slice_textures; basisu_backend_output m_uastc_backend_output; // The amount the HDR input has to be scaled up in case it had to be rescaled to fit into half floats. - float m_hdr_image_scale; - + float m_hdr_image_scale; + // The upconversion multiplier used to load LDR images in HDR mode. float m_ldr_to_hdr_upconversion_nit_multiplier; - + // True if any loaded source images were LDR and upconverted to HDR. bool m_upconverted_any_ldr_images; @@ -701,48 +893,20 @@ namespace basisu error_code encode_slices_to_astc_6x6_hdr(); error_code encode_slices_to_uastc_4x4_hdr(); error_code encode_slices_to_uastc_4x4_ldr(); + error_code encode_slices_to_xuastc_or_astc_ldr(); bool generate_mipmaps(const imagef& img, basisu::vector& mips, bool has_alpha); bool generate_mipmaps(const image &img, basisu::vector &mips, bool has_alpha); bool validate_texture_type_constraints(); bool validate_ktx2_constraints(); bool get_dfd(uint8_vec& dfd, const basist::ktx2_header& hdr); bool create_ktx2_file(); - void pick_format_mode(); + bool pick_format_mode(); - uint32_t get_block_width() const - { - if (m_params.m_hdr) - { - switch (m_params.m_hdr_mode) - { - case hdr_modes::cASTC_HDR_6X6: - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - return 6; - default: - break; - } - } - return 4; - } - - uint32_t get_block_height() const - { - if (m_params.m_hdr) - { - switch (m_params.m_hdr_mode) - { - case hdr_modes::cASTC_HDR_6X6: - case hdr_modes::cASTC_HDR_6X6_INTERMEDIATE: - return 6; - default: - break; - } - } - return 4; - } + uint32_t get_block_width() const { return m_fmt_mode_block_width; } + uint32_t get_block_height() const { return m_fmt_mode_block_height; } }; - - // Alternative simple C-style wrapper API around the basis_compressor class. + + // Alternative simple C-style wrapper API around the basis_compressor class. // This doesn't expose every encoder feature, but it's enough to get going. // Important: basisu_encoder_init() MUST be called first before calling these functions. // @@ -751,16 +915,22 @@ namespace basisu // OR // pImageRGBA: pointer to a 32-bpp RGBx or RGBA raster image, R first in memory, A last. Top scanline first in memory. // width/height/pitch_in_pixels: dimensions of pImageRGBA - // - // flags_and_quality: Combination of the above flags logically OR'd with the ETC1S or UASTC level, i.e. "cFlagSRGB | cFlagGenMipsClamp | cFlagThreaded | 128" or "cFlagSRGB | cFlagGenMipsClamp | cFlagUASTC | cFlagThreaded | cPackUASTCLevelDefault". + // + // flags_and_quality: Combination of the above flags logically OR'd with the ETC1S or UASTC quality or effort level. + // Note: basis_compress2() variants below accept the new-style "quality_level" (0-100) and "effort_level" (0-10) parameters instead of packing them into flags_and_quality. // In ETC1S mode, the lower 8-bits are the ETC1S quality level which ranges from [1,255] (higher=better quality/larger files) - // In UASTC mode, the lower 8-bits are the UASTC LDR/HDR pack level (see cPackUASTCLevelFastest, etc.). Fastest/lowest quality is 0, so be sure to set it correctly. Valid values are [0,4] for both LDR/HDR. - // In UASTC mode, be sure to set this, otherwise it defaults to 0 (fastest/lowest quality). - // - // uastc_rdo_quality: Float UASTC RDO quality level (0=no change, higher values lower quality but increase compressibility, initially try .5-1.5) - // + // In UASTC LDR 4x4 mode, the lower 8-bits are the UASTC LDR/HDR pack or effort level (see cPackUASTCLevelFastest to cPackUASTCLevelVerySlow). Fastest/lowest quality is 0, so be sure to set it correctly. Valid values are [0,4] for both LDR/HDR. + // In UASTC HDR 4x4 mode, the lower 8-bits are the codec's effort level. Valid range is [uastc_hdr_4x4_codec_options::cMinLevel, uastc_hdr_4x4_codec_options::cMaxLevel]. Higher=better quality, but slower. + // In RDO ASTC HDR 6x6/UASTC HDR 6x6 mode, the lower 8-bits are the codec's effort level. Valid range is [0,astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL]. Higher levels=better quality, but slower. + // In XUASTC/ASTC LDR 4x4-12x12 mode, the lower 8-bits are the compressor's effort level from [0,10] (astc_ldr_t::EFFORT_LEVEL_MIN, astc_ldr_t::EFFORT_LEVEL_MAX). + // + // float uastc_rdo_or_dct_quality: + // UASTC LDR 4x4 RDO quality level: RDO lambda setting - 0=no change/highest quality. Higher values lower quality but increase compressibility, initially try .5-1.5. + // RDO ASTC 6x6 HDR/UASTC 6x6 HDR: RDO lambda setting. 0=no change/highest quality. Higher values lower quality but increase compressibility, initially try 250-2000 (HDR) or 1000-10000 (LDR/SDR inputs upconverted to HDR). + // In XUASTC/ASTC LDR 4x4-12x12 mode, this is the [1,100] weight grid DCT quality level. + // // pSize: Returns the output data's compressed size in bytes - // + // // Return value is the compressed .basis or .ktx2 file data, or nullptr on failure. Must call basis_free() to free it. enum { @@ -769,36 +939,63 @@ namespace basisu cFlagDebug = 1 << 10, // enable debug output cFlagKTX2 = 1 << 11, // generate a KTX2 file - cFlagKTX2UASTCSuperCompression = 1 << 12, // use KTX2 Zstd supercompression on UASTC files + cFlagKTX2UASTCSuperCompression = 1 << 12, // use KTX2 Zstd supercompression on non-supercompressed formats that support it. - cFlagSRGB = 1 << 13, // input texture is sRGB, use perceptual colorspace metrics, also use sRGB filtering during mipmap gen, and also sets KTX2 output transfer func to sRGB + cFlagSRGB = 1 << 13, // input texture is sRGB, use perceptual colorspace metrics, also use sRGB filtering during mipmap gen, and also sets KTX2/.basis output transfer func to sRGB cFlagGenMipsClamp = 1 << 14, // generate mipmaps with clamp addressing cFlagGenMipsWrap = 1 << 15, // generate mipmaps with wrap addressing - + cFlagYFlip = 1 << 16, // flip source image on Y axis before compression - - cFlagUASTCRDO = 1 << 17, // use RDO postprocessing when generating UASTC files (must set uastc_rdo_quality to the quality scalar) - + + // Note 11/18/2025: cFlagUASTCRDO flag is now ignored. Now if uastc_rdo_or_dct_quality>0 in UASTC LDR 4x4 mode, you automatically get RDO. + //cFlagUASTCRDO = 1 << 17, // use RDO postprocessing when generating UASTC LDR 4x4 files (must set uastc_rdo_or_dct_quality to the quality scalar) + cFlagPrintStats = 1 << 18, // print image stats to stdout cFlagPrintStatus = 1 << 19, // print status to stdout + + cFlagDebugImages = 1 << 20, // enable debug image generation (for development, slower) - cFlagDebugImages = 1 << 20, // enable status output - - cFlagREC2020 = 1 << 21, // ASTC 6x6 modes: treat input as REC 2020 vs. the default 709 - + cFlagREC2020 = 1 << 21, // treat input as REC 2020 vs. the default 709 (for codecs that support this, currently UASTC HDR and ASTC 6x6), bit is always placed into KTX2 DFD + cFlagValidateOutput = 1 << 22, // transcode the output after encoding for testing + + // XUASTC LDR profile: full arith, hybrid or full zstd (see basist::astc_ldr_t::xuastc_ldr_syntax) + cFlagXUASTCLDRSyntaxFullArith = 0 << 23, + cFlagXUASTCLDRSyntaxHybrid = 1 << 23, + cFlagXUASTCLDRSyntaxFullZStd = 2 << 23, + + cFlagXUASTCLDRSyntaxShift = 23, + cFlagXUASTCLDRSyntaxMask = 3, + + // Texture Type: 2D, 2D Array, Cubemap Array, or Texture Video (see enum basis_texture_type). Defaults to plain 2D. + cFlagTextureType2D = 0 << 25, + cFlagTextureType2DArray = 1 << 25, + cFlagTextureTypeCubemapArray = 2 << 25, + cFlagTextureTypeVideoFrames = 3 << 25, + + cFlagTextureTypeShift = 25, + cFlagTextureTypeMask = 3, }; - // This function accepts an array of source images. + void* basis_compress_internal( + basist::basis_tex_format mode, + const basisu::vector* pSource_images, + const basisu::vector* pSource_images_hdr, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats, + int quality_level = -1, int effort_level = -1); + + // This function accepts an array of source images. // If more than one image is provided, it's assumed the images form a mipmap pyramid and automatic mipmap generation is disabled. - // Returns a pointer to the compressed .basis or .ktx2 file data. *pSize is the size of the compressed data. + // Returns a pointer to the compressed .basis or .ktx2 file data. *pSize is the size of the compressed data. // Important: The returned block MUST be manually freed using basis_free_data(). // basisu_encoder_init() MUST be called first! - // LDR version. To compress the LDR source image as HDR: Use the cFlagHDR flag. + // LDR version. void* basis_compress( basist::basis_tex_format mode, const basisu::vector &source_images, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats = nullptr); @@ -807,7 +1004,7 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const basisu::vector& source_images_hdr, - uint32_t flags_and_quality, float lambda, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, size_t* pSize, image_stats* pStats = nullptr); @@ -816,7 +1013,30 @@ namespace basisu void* basis_compress( basist::basis_tex_format mode, const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, - uint32_t flags_and_quality, float uastc_rdo_quality, + uint32_t flags_and_quality, float uastc_rdo_or_dct_quality, + size_t* pSize, + image_stats* pStats = nullptr); + + // basis_compress2() variants accept the new unified quality_level and effort_level parameters instead of the old flags/float uastc_rdo_or_dct_quality parameter. + // quality_level must be [0,100], effort_level [0,10]. + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats = nullptr); + + void* basis_compress2( + basist::basis_tex_format mode, + const basisu::vector& source_images_hdr, + uint32_t flags_and_quality, int quality_level, int effort_level, + size_t* pSize, + image_stats* pStats = nullptr); + + void* basis_compress2( + basist::basis_tex_format mode, + const uint8_t* pImageRGBA, uint32_t width, uint32_t height, uint32_t pitch_in_pixels, + uint32_t flags_and_quality, int quality_level, int effort_level, size_t* pSize, image_stats* pStats = nullptr); @@ -841,7 +1061,7 @@ namespace basisu double m_basis_bits_per_texel; bool m_any_source_image_has_alpha; - parallel_results() + parallel_results() { clear(); } @@ -857,7 +1077,7 @@ namespace basisu m_any_source_image_has_alpha = false; } }; - + // Compresses an array of input textures across total_threads threads using the basis_compressor class. // Compressing multiple textures at a time is substantially more efficient than just compressing one at a time. // total_threads must be >= 1. @@ -865,5 +1085,6 @@ namespace basisu uint32_t total_threads, const basisu::vector ¶ms_vec, basisu::vector< parallel_results > &results_vec); - + } // namespace basisu + diff --git a/encoder/basisu_enc.cpp b/encoder/basisu_enc.cpp index cccf66c..841209b 100644 --- a/encoder/basisu_enc.cpp +++ b/encoder/basisu_enc.cpp @@ -1,5 +1,5 @@ // basisu_enc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ #include "basisu_opencl.h" #include "basisu_uastc_hdr_4x4_enc.h" #include "basisu_astc_hdr_6x6_enc.h" +#include "basisu_astc_ldr_common.h" +#include "basisu_astc_ldr_encode.h" #include @@ -58,7 +60,7 @@ namespace basisu #endif fast_linear_to_srgb g_fast_linear_to_srgb; - + uint8_t g_hamming_dist[256] = { 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, @@ -81,7 +83,7 @@ namespace basisu // This is a Public Domain 8x8 font from here: // https://github.com/dhepper/font8x8/blob/master/font8x8_basic.h - const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8] = + const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8] = { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // U+0020 ( ) { 0x18, 0x3C, 0x3C, 0x18, 0x18, 0x00, 0x18, 0x00}, // U+0021 (!) @@ -181,9 +183,17 @@ namespace basisu { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} // U+007F }; + float g_srgb_to_linear_table[256]; + + void init_srgb_to_linear_table() + { + for (int i = 0; i < 256; ++i) + g_srgb_to_linear_table[i] = srgb_to_linear((float)i * (1.0f / 255.0f)); + } + bool g_library_initialized; std::mutex g_encoder_init_mutex; - + // Encoder library initialization (just call once at startup) bool basisu_encoder_init(bool use_opencl, bool opencl_force_serialization) { @@ -193,7 +203,7 @@ namespace basisu return true; detect_sse41(); - + basist::basisu_transcoder_init(); pack_etc1_solid_color_init(); //uastc_init(); @@ -210,7 +220,11 @@ namespace basisu astc_hdr_enc_init(); basist::bc6h_enc_init(); astc_6x6_hdr::global_init(); + astc_ldr::global_init(); + astc_ldr::encoder_init(); + init_srgb_to_linear_table(); + g_library_initialized = true; return true; } @@ -221,7 +235,7 @@ namespace basisu g_library_initialized = false; } - + void error_vprintf(const char* pFmt, va_list args) { const uint32_t BUF_SIZE = 256; @@ -238,10 +252,12 @@ namespace basisu return; } + fflush(stdout); + if (total_chars >= (int)BUF_SIZE) { basisu::vector var_buf(total_chars + 1); - + va_copy(args_copy, args); int total_chars_retry = vsnprintf(var_buf.data(), var_buf.size(), pFmt, args_copy); va_end(args_copy); @@ -277,6 +293,7 @@ namespace basisu void platform_sleep(uint32_t ms) { // TODO + BASISU_NOTE_UNUSED(ms); } #endif @@ -316,7 +333,7 @@ namespace basisu #else #error TODO #endif - + interval_timer::interval_timer() : m_start_time(0), m_stop_time(0), m_started(false), m_stopped(false) { if (!g_timer_freq) @@ -350,7 +367,7 @@ namespace basisu timer_ticks delta = stop_time - m_start_time; return delta * g_timer_freq; } - + void interval_timer::init() { if (!g_timer_freq) @@ -377,7 +394,7 @@ namespace basisu return ticks * g_timer_freq; } - // Note this is linear<->sRGB, NOT REC709 which uses slightly different equations/transfer functions. + // Note this is linear<->sRGB, NOT REC709 which uses slightly different equations/transfer functions. // However the gamuts/white points of REC709 and sRGB are the same. float linear_to_srgb(float l) { @@ -387,7 +404,7 @@ namespace basisu else return saturate(1.055f * powf(l, 1.0f / 2.4f) - .055f); } - + float srgb_to_linear(float s) { assert(s >= 0.0f && s <= 1.0f); @@ -396,21 +413,21 @@ namespace basisu else return saturate(powf((s + .055f) * (1.0f / 1.055f), 2.4f)); } - + const uint32_t MAX_32BIT_ALLOC_SIZE = 250000000; - + bool load_tga(const char* pFilename, image& img) { int w = 0, h = 0, n_chans = 0; uint8_t* pImage_data = read_tga(pFilename, w, h, n_chans); - + if ((!pImage_data) || (!w) || (!h) || ((n_chans != 3) && (n_chans != 4))) { error_printf("Failed loading .TGA image \"%s\"!\n", pFilename); if (pImage_data) free(pImage_data); - + return false; } @@ -426,7 +443,7 @@ namespace basisu return false; } } - + img.resize(w, h); const uint8_t *pSrc = pImage_data; @@ -469,7 +486,7 @@ namespace basisu { interval_timer tm; tm.start(); - + if (!buf_size) return false; @@ -488,7 +505,7 @@ namespace basisu return true; } - + bool load_png(const char* pFilename, image& img) { uint8_vec buffer; @@ -507,9 +524,9 @@ namespace basisu uint8_t *pImage_data = jpgd::decompress_jpeg_image_from_file(pFilename, &width, &height, &actual_comps, 4, jpgd::jpeg_decoder::cFlagLinearChromaFiltering); if (!pImage_data) return false; - + img.init(pImage_data, width, height, 4); - + free(pImage_data); return true; @@ -642,7 +659,7 @@ namespace basisu dst[2] = basist::half_to_float(pSrc_pixel[2]); dst[3] = basist::half_to_float(pSrc_pixel[3]); } - + pSrc_image_h += (width * 4); } @@ -722,7 +739,7 @@ namespace basisu return ((strcasecmp(pExt, "hdr") == 0) || (strcasecmp(pExt, "exr") == 0)); } - + // TODO: move parameters to struct, add a HDR clean flag to eliminate NaN's/Inf's bool load_image_hdr(const char* pFilename, imagef& img, bool ldr_srgb_to_linear, float linear_nit_multiplier, float ldr_black_bias) { @@ -740,7 +757,7 @@ namespace basisu return false; return true; } - + if (strcasecmp(pExt, "exr") == 0) { int n_chans = 0; @@ -760,12 +777,12 @@ namespace basisu return true; } - + bool save_png(const char* pFilename, const image &img, uint32_t image_save_flags, uint32_t grayscale_comp) { if (!img.get_total_pixels()) return false; - + void* pPNG_data = nullptr; size_t PNG_data_size = 0; @@ -783,7 +800,7 @@ namespace basisu else { bool has_alpha = false; - + if ((image_save_flags & cImageSaveIgnoreAlpha) == 0) has_alpha = img.has_alpha(); @@ -800,7 +817,7 @@ namespace basisu pDst[0] = pSrc->r; pDst[1] = pSrc->g; pDst[2] = pSrc->b; - + pSrc++; pDst += 3; } @@ -824,10 +841,35 @@ namespace basisu } free(pPNG_data); - + return status; } + bool save_qoi(const char* pFilename, const image& img, uint32_t qoi_colorspace) + { + assert(img.get_width() && img.get_height()); + + qoi_desc desc; + clear_obj(desc); + + desc.width = img.get_width(); + desc.height = img.get_height(); + desc.channels = 4; + desc.colorspace = (uint8_t)qoi_colorspace; + + int out_len = 0; + void* pData = qoi_encode(img.get_ptr(), &desc, &out_len); + if ((!pData) || (!out_len)) + return false; + + const bool status = write_data_to_file(pFilename, pData, out_len); + + QOI_FREE(pData); + pData = nullptr; + + return status; + } + bool read_file_to_vec(const char* pFilename, uint8_vec& data) { FILE* pFile = nullptr; @@ -838,7 +880,7 @@ namespace basisu #endif if (!pFile) return false; - + fseek(pFile, 0, SEEK_END); #ifdef _WIN32 int64_t filesize = _ftelli64(pFile); @@ -909,7 +951,7 @@ namespace basisu return false; } fseek(pFile, 0, SEEK_SET); - + if (fread(pData, 1, (size_t)len, pFile) != (size_t)len) { fclose(pFile); @@ -942,19 +984,20 @@ namespace basisu return fclose(pFile) != EOF; } - + bool image_resample(const image &src, image &dst, bool srgb, - const char *pFilter, float filter_scale, + const char *pFilter, float filter_scale, bool wrapping, - uint32_t first_comp, uint32_t num_comps) + uint32_t first_comp, uint32_t num_comps, + float filter_scale_y) { assert((first_comp + num_comps) <= 4); const int cMaxComps = 4; - + const uint32_t src_w = src.get_width(), src_h = src.get_height(); const uint32_t dst_w = dst.get_width(), dst_h = dst.get_height(); - + if (maximum(src_w, src_h) > BASISU_RESAMPLER_MAX_DIMENSION) { printf("Image is too large!\n"); @@ -963,17 +1006,19 @@ namespace basisu if (!src_w || !src_h || !dst_w || !dst_h) return false; - + if ((num_comps < 1) || (num_comps > cMaxComps)) return false; - + if ((minimum(dst_w, dst_h) < 1) || (maximum(dst_w, dst_h) > BASISU_RESAMPLER_MAX_DIMENSION)) { printf("Image is too large!\n"); return false; } - if ((src_w == dst_w) && (src_h == dst_h)) + if ( (src_w == dst_w) && (src_h == dst_h) && + (filter_scale == 1.0f) && + ((filter_scale_y < 0.0f) || (filter_scale_y == 1.0f)) ) { dst = src; return true; @@ -997,17 +1042,19 @@ namespace basisu std::vector samples[cMaxComps]; Resampler *resamplers[cMaxComps]; - + resamplers[0] = new Resampler(src_w, src_h, dst_w, dst_h, wrapping ? Resampler::BOUNDARY_WRAP : Resampler::BOUNDARY_CLAMP, 0.0f, 1.0f, - pFilter, nullptr, nullptr, filter_scale, filter_scale, 0, 0); + pFilter, nullptr, nullptr, + filter_scale, (filter_scale_y >= 0.0f) ? filter_scale_y : filter_scale, 0, 0); samples[0].resize(src_w); for (uint32_t i = 1; i < num_comps; ++i) { resamplers[i] = new Resampler(src_w, src_h, dst_w, dst_h, wrapping ? Resampler::BOUNDARY_WRAP : Resampler::BOUNDARY_CLAMP, 0.0f, 1.0f, - pFilter, resamplers[0]->get_clist_x(), resamplers[0]->get_clist_y(), filter_scale, filter_scale, 0, 0); + pFilter, resamplers[0]->get_clist_x(), resamplers[0]->get_clist_y(), + filter_scale, (filter_scale_y >= 0.0f) ? filter_scale_y : filter_scale, 0, 0); samples[i].resize(src_w); } @@ -1057,7 +1104,7 @@ namespace basisu break; const bool linear_flag = !srgb || (comp_index == 3); - + color_rgba *pDst = &dst(0, dst_y); for (uint32_t x = 0; x < dst_w; x++) @@ -1090,7 +1137,7 @@ namespace basisu return true; } - bool image_resample(const imagef& src, imagef& dst, + bool image_resample(const imagef& src, imagef& dst, const char* pFilter, float filter_scale, bool wrapping, uint32_t first_comp, uint32_t num_comps) @@ -1183,7 +1230,7 @@ namespace basisu const float* pOutput_samples = resamplers[c]->get_line(); if (!pOutput_samples) break; - + vec4F* pDst = &dst(0, dst_y); for (uint32_t x = 0; x < dst_w; x++) @@ -1216,9 +1263,9 @@ namespace basisu A[0].m_key = 1; return; } - + A[0].m_key += A[1].m_key; - + int s = 2, r = 0, next; for (next = 1; next < (num_syms - 1); ++next) { @@ -1310,7 +1357,7 @@ namespace basisu for (i = 0; i < num_syms; i++) { uint32_t freq = pSyms0[i].m_key; - + // We scale all input frequencies to 16-bits. assert(freq <= UINT16_MAX); @@ -1501,7 +1548,7 @@ namespace basisu uint32_t total_used = tab.get_total_used_codes(); put_bits(total_used, cHuffmanMaxSymsLog2); - + if (!total_used) return 0; @@ -1565,7 +1612,7 @@ namespace basisu const uint32_t l = syms[i] & 63, e = syms[i] >> 6; put_code(l, ct); - + if (l == cHuffmanSmallZeroRunCode) put_bits(e, cHuffmanSmallZeroRunExtraBits); else if (l == cHuffmanBigZeroRunCode) @@ -1592,7 +1639,7 @@ namespace basisu huffman_encoding_table etab; etab.init(h, 16); - + { bitwise_coder c; c.init(1024); @@ -1727,9 +1774,9 @@ namespace basisu // We now have chosen an entry to place in the picked list, now determine which side it goes on. const uint32_t entry_to_move = m_entries_to_do[best_entry]; - + float side = pick_side(num_syms, entry_to_move, pDist_func, pCtx, dist_func_weight); - + // Put entry_to_move either on the "left" or "right" side of the picked entries if (side <= 0) m_entries_picked.push_back(entry_to_move); @@ -1832,7 +1879,7 @@ namespace basisu } return which_side; } - + void image_metrics::calc(const imagef& a, const imagef& b, uint32_t first_chan, uint32_t total_chans, bool avg_comp_error, bool log) { assert((first_chan < 4U) && (first_chan + total_chans <= 4U)); @@ -1843,16 +1890,19 @@ namespace basisu double max_e = -1e+30f; double sum = 0.0f, sum_sqr = 0.0f; + m_width = width; + m_height = height; + m_has_neg = false; m_any_abnormal = false; m_hf_mag_overflow = false; - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) { const vec4F& ca = a(x, y), &cb = b(x, y); - + if (total_chans) { for (uint32_t c = 0; c < total_chans; c++) @@ -1867,7 +1917,7 @@ namespace basisu if (std::isinf(fa) || std::isinf(fb) || std::isnan(fa) || std::isnan(fb)) m_any_abnormal = true; - + const double delta = fabs(fa - fb); max_e = basisu::maximum(max_e, delta); @@ -1902,10 +1952,10 @@ namespace basisu } double ca_l = get_luminance(ca), cb_l = get_luminance(cb); - + double delta = fabs(ca_l - cb_l); max_e = basisu::maximum(max_e, delta); - + if (log) { double log2_delta = log2(basisu::maximum(0.0f, ca_l) + 1.0f) - log2(basisu::maximum(0.0f, cb_l) + 1.0f); @@ -1931,7 +1981,7 @@ namespace basisu m_mean = (float)(sum / total_values); m_mean_squared = (float)(sum_sqr / total_values); m_rms = (float)sqrt(sum_sqr / total_values); - + const double max_val = 1.0f; m_psnr = m_rms ? (float)clamp(log10(max_val / m_rms) * 20.0f, 0.0f, 1000.0f) : 1000.0f; } @@ -1944,12 +1994,15 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; uint_vec hist(65536); - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -1960,7 +2013,7 @@ namespace basisu { if ((ca[i] < 0.0f) || (cb[i] < 0.0f)) m_has_neg = true; - + if ((fabs(ca[i]) > basist::MAX_HALF_FLOAT) || (fabs(cb[i]) > basist::MAX_HALF_FLOAT)) m_hf_mag_overflow = true; @@ -2010,10 +2063,13 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; - + double sum = 0.0f, sum2 = 0.0f; m_max = 0; @@ -2050,7 +2106,7 @@ namespace basisu } // x } // y - + double total_values = (double)width * (double)height; if (avg_comp_error) total_values *= (double)clamp(total_chans, 1, 4); @@ -2069,12 +2125,17 @@ namespace basisu const uint32_t width = basisu::minimum(a.get_width(), b.get_width()); const uint32_t height = basisu::minimum(a.get_height(), b.get_height()); + m_width = width; + m_height = height; + double hist[256]; clear_obj(hist); m_has_neg = false; m_any_abnormal = false; m_hf_mag_overflow = false; + m_sum_a = 0; + m_sum_b = 0; for (uint32_t y = 0; y < height; y++) { @@ -2085,7 +2146,11 @@ namespace basisu if (total_chans) { for (uint32_t c = 0; c < total_chans; c++) + { hist[iabs(ca[first_chan + c] - cb[first_chan + c])]++; + m_sum_a += ca[first_chan + c]; + m_sum_b += cb[first_chan + c]; + } } else { @@ -2093,6 +2158,12 @@ namespace basisu hist[iabs(ca.get_601_luma() - cb.get_601_luma())]++; else hist[iabs(ca.get_709_luma() - cb.get_709_luma())]++; + + for (uint32_t c = 0; c < 3; c++) + { + m_sum_a += ca[c]; + m_sum_b += cb[c]; + } } } } @@ -2168,63 +2239,7 @@ namespace basisu } } - uint32_t hash_hsieh(const uint8_t *pBuf, size_t len) - { - if (!pBuf || !len) - return 0; - - uint32_t h = static_cast(len); - - const uint32_t bytes_left = len & 3; - len >>= 2; - - while (len--) - { - const uint16_t *pWords = reinterpret_cast(pBuf); - - h += pWords[0]; - - const uint32_t t = (pWords[1] << 11) ^ h; - h = (h << 16) ^ t; - - pBuf += sizeof(uint32_t); - - h += h >> 11; - } - - switch (bytes_left) - { - case 1: - h += *reinterpret_cast(pBuf); - h ^= h << 10; - h += h >> 1; - break; - case 2: - h += *reinterpret_cast(pBuf); - h ^= h << 11; - h += h >> 17; - break; - case 3: - h += *reinterpret_cast(pBuf); - h ^= h << 16; - h ^= (static_cast(pBuf[sizeof(uint16_t)])) << 18; - h += h >> 11; - break; - default: - break; - } - - h ^= h << 3; - h += h >> 5; - h ^= h << 4; - h += h >> 17; - h ^= h << 25; - h += h >> 6; - - return h; - } - - job_pool::job_pool(uint32_t num_threads) : + job_pool::job_pool(uint32_t num_threads) : m_num_active_jobs(0) { m_kill_flag.store(false); @@ -2246,13 +2261,13 @@ namespace basisu job_pool::~job_pool() { debug_printf("job_pool::~job_pool\n"); - + // Notify all workers that they need to die right now. { std::lock_guard lk(m_mutex); m_kill_flag.store(true); } - + m_has_work.notify_all(); #ifdef __EMSCRIPTEN__ @@ -2262,7 +2277,7 @@ namespace basisu break; std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - + // At this point all worker threads should be exiting or exited. // We could call detach(), but this seems to just call join() anyway. #endif @@ -2271,7 +2286,7 @@ namespace basisu for (uint32_t i = 0; i < m_threads.size(); i++) m_threads[i].join(); } - + void job_pool::add_job(const std::function& job) { std::unique_lock lock(m_mutex); @@ -2291,7 +2306,7 @@ namespace basisu std::unique_lock lock(m_mutex); m_queue.emplace_back(std::move(job)); - + const size_t queue_size = m_queue.size(); lock.unlock(); @@ -2340,7 +2355,7 @@ namespace basisu //debug_printf("job_pool::job_thread: starting %u\n", index); m_num_active_workers.fetch_add(1); - + while (!m_kill_flag) { std::unique_lock lock(m_mutex); @@ -2376,9 +2391,9 @@ namespace basisu --m_num_active_jobs; - // Now check if there are no more jobs remaining. + // Now check if there are no more jobs remaining. const bool all_done = m_queue.empty() && !m_num_active_jobs; - + lock.unlock(); if (all_done) @@ -2439,7 +2454,7 @@ namespace basisu // Simple validation if ((hdr.m_cmap != 0) && (hdr.m_cmap != 1)) return nullptr; - + if (hdr.m_cmap) { if ((hdr.m_cmap_bpp == 0) || (hdr.m_cmap_bpp > 32)) @@ -2598,13 +2613,13 @@ namespace basisu bytes_remaining += bytes_to_skip; } } - + width = hdr.m_width; height = hdr.m_height; const uint32_t source_pitch = width * tga_bytes_per_pixel; const uint32_t dest_pitch = width * n_chans; - + uint8_t *pImage = (uint8_t *)malloc(dest_pitch * height); if (!pImage) return nullptr; @@ -2626,7 +2641,7 @@ namespace basisu int pixels_remaining = width; uint8_t *pDst = &input_line_buf[0]; - do + do { if (!run_remaining) { @@ -2811,7 +2826,7 @@ namespace basisu if (!filedata.size() || (filedata.size() > UINT32_MAX)) return nullptr; - + return read_tga(&filedata[0], (uint32_t)filedata.size(), width, height, n_chans); } @@ -2958,13 +2973,13 @@ namespace basisu if (cur_line.size() < 3) return false; - + if (!is_x && !is_y) return false; comp[d] = is_x ? 0 : 1; dir[d] = (is_neg_x || is_neg_y) ? -1 : 1; - + uint32_t& dim = d ? minor_dim : major_dim; cur_line.erase(0, 3); @@ -3002,7 +3017,7 @@ namespace basisu if ((dim < 1) || (dim > MAX_SUPPORTED_DIM)) return false; } - + // temp image: width=minor, height=major img.resize(minor_dim, major_dim); @@ -3030,7 +3045,7 @@ namespace basisu } else { - // c[0]/red is 2.Check GB and E for validity. + // c[0]/red is 2.Check GB and E for validity. color_rgba c; memcpy(&c, &filedata[cur_ofs], 4); @@ -3152,7 +3167,7 @@ namespace basisu // width=minor axis dimension // height=major axis dimension // in file, pixels are emitted in minor order, them major (so major=scanlines in the file) - + imagef final_img; if (comp[0] == 0) // if major axis is X final_img.resize(major_dim, minor_dim); @@ -3169,10 +3184,10 @@ namespace basisu uint32_t dst_x = 0, dst_y = 0; // is the minor dim output x? - if (comp[1] == 0) + if (comp[1] == 0) { // minor axis is x, major is y - + // is minor axis (which is output x) flipped? if (dir[1] < 0) dst_x = minor_dim - 1 - minor_iter; @@ -3231,7 +3246,7 @@ namespace basisu return buf; } - + static uint8_vec& append_string(uint8_vec& buf, const std::string& str) { if (!str.size()) @@ -3248,7 +3263,7 @@ namespace basisu if (max_v < 1e-32f) rgbe.clear(); - else + else { int e; const float scale = frexp(max_v, &e) * 256.0f / max_v; @@ -3261,14 +3276,14 @@ namespace basisu const bool RGBE_FORCE_RAW = false; const bool RGBE_FORCE_OLD_CRUNCH = false; // note must readers (particularly stb_image.h's) don't properly support this, when they should - + bool write_rgbe(uint8_vec &file_data, imagef& img, rgbe_header_info& hdr_info) { if (!img.get_width() || !img.get_height()) return false; const uint32_t width = img.get_width(), height = img.get_height(); - + file_data.resize(0); file_data.reserve(1024 + img.get_width() * img.get_height() * 4); @@ -3301,7 +3316,7 @@ namespace basisu { int prev_r = -1, prev_g = -1, prev_b = -1, prev_e = -1; uint32_t cur_run_len = 0; - + for (uint32_t x = 0; x < width; x++) { color_rgba rgbe; @@ -3314,7 +3329,7 @@ namespace basisu // this ensures rshift stays 0, it's lame but this path is only for testing readers color_rgba f(1, 1, 1, cur_run_len - 1); append_vector(file_data, (const uint8_t*)&f, sizeof(f)); - append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); + append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); cur_run_len = 0; } } @@ -3324,12 +3339,12 @@ namespace basisu { color_rgba f(1, 1, 1, cur_run_len); append_vector(file_data, (const uint8_t*)&f, sizeof(f)); - + cur_run_len = 0; } - + append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); - + prev_r = rgbe[0]; prev_g = rgbe[1]; prev_b = rgbe[2]; @@ -3354,7 +3369,7 @@ namespace basisu { color_rgba rgbe(2, 2, width >> 8, width & 0xFF); append_vector(file_data, (const uint8_t*)&rgbe, sizeof(rgbe)); - + for (uint32_t x = 0; x < width; x++) { float2rgbe(rgbe, img(x, y)); @@ -3366,7 +3381,7 @@ namespace basisu for (uint32_t c = 0; c < 4; c++) { int raw_ofs = -1; - + uint32_t x = 0; while (x < width) { @@ -3381,7 +3396,7 @@ namespace basisu break; run_len++; } - + const uint32_t cost_to_keep_raw = ((raw_ofs != -1) ? 0 : 1) + run_len; // 0 or 1 bytes to start a raw run, then the repeated bytes issued as raw const uint32_t cost_to_take_run = 2 + 1; // 2 bytes to issue the RLE, then 1 bytes to start whatever follows it (raw or RLE) @@ -3405,7 +3420,7 @@ namespace basisu raw_ofs = -1; file_data.push_back(cur_byte); - + x++; } } // x @@ -3424,7 +3439,7 @@ namespace basisu return false; return write_vec_to_file(pFilename, file_data); } - + bool read_exr(const char* pFilename, imagef& img, int& n_chans) { n_chans = 0; @@ -3432,7 +3447,7 @@ namespace basisu int width = 0, height = 0; float* out_rgba = nullptr; const char* err = nullptr; - + int status = LoadEXRWithLayer(&out_rgba, &width, &height, pFilename, nullptr, &err, &n_chans); if (status != 0) { @@ -3451,7 +3466,7 @@ namespace basisu } img.resize(width, height); - + if (n_chans == 1) { const float* pSrc = out_rgba; @@ -3505,16 +3520,16 @@ namespace basisu { assert((n_chans == 1) || (n_chans == 3) || (n_chans == 4)); - const bool linear_hint = (flags & WRITE_EXR_LINEAR_HINT) != 0, + const bool linear_hint = (flags & WRITE_EXR_LINEAR_HINT) != 0, store_float = (flags & WRITE_EXR_STORE_FLOATS) != 0, no_compression = (flags & WRITE_EXR_NO_COMPRESSION) != 0; - + const uint32_t width = img.get_width(), height = img.get_height(); assert(width && height); - + if (!width || !height) return false; - + float_vec layers[4]; float* image_ptrs[4]; for (uint32_t c = 0; c < n_chans; c++) @@ -3543,7 +3558,7 @@ namespace basisu assert(0); return false; } - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -3567,7 +3582,7 @@ namespace basisu image.height = height; header.num_channels = n_chans; - + header.channels = (EXRChannelInfo*)calloc(header.num_channels, sizeof(EXRChannelInfo)); // Must be (A)BGR order, since most of EXR viewers expect this channel order. @@ -3578,37 +3593,37 @@ namespace basisu c = "BGR"[i]; else if (n_chans == 4) c = "ABGR"[i]; - + header.channels[i].name[0] = c; header.channels[i].name[1] = '\0'; header.channels[i].p_linear = linear_hint; } - + header.pixel_types = (int*)calloc(header.num_channels, sizeof(int)); header.requested_pixel_types = (int*)calloc(header.num_channels, sizeof(int)); - + if (!no_compression) header.compression_type = TINYEXR_COMPRESSIONTYPE_ZIP; - for (int i = 0; i < header.num_channels; i++) + for (int i = 0; i < header.num_channels; i++) { // pixel type of input image - header.pixel_types[i] = TINYEXR_PIXELTYPE_FLOAT; + header.pixel_types[i] = TINYEXR_PIXELTYPE_FLOAT; // pixel type of output image to be stored in .EXR - header.requested_pixel_types[i] = store_float ? TINYEXR_PIXELTYPE_FLOAT : TINYEXR_PIXELTYPE_HALF; + header.requested_pixel_types[i] = store_float ? TINYEXR_PIXELTYPE_FLOAT : TINYEXR_PIXELTYPE_HALF; } const char* pErr_msg = nullptr; int ret = SaveEXRImageToFile(&image, &header, pFilename, &pErr_msg); - if (ret != TINYEXR_SUCCESS) + if (ret != TINYEXR_SUCCESS) { error_printf("Save EXR err: %s\n", pErr_msg); FreeEXRErrorMessage(pErr_msg); } - + free(header.channels); free(header.pixel_types); free(header.requested_pixel_types); @@ -3622,7 +3637,7 @@ namespace basisu va_list args; va_start(args, pFmt); -#ifdef _WIN32 +#ifdef _WIN32 vsprintf_s(buf, sizeof(buf), pFmt, args); #else vsnprintf(buf, sizeof(buf), pFmt, args); @@ -3647,7 +3662,7 @@ namespace basisu for (uint32_t x = 0; x < 8; x++) { const uint32_t q = row_bits & (1 << x); - + const color_rgba* pColor = q ? &fg : pBG; if (!pColor) continue; @@ -3667,8 +3682,8 @@ namespace basisu } } } - - // Very basic global Reinhard tone mapping, output converted to sRGB with no dithering, alpha is carried through unchanged. + + // Very basic global Reinhard tone mapping, output converted to sRGB with no dithering, alpha is carried through unchanged. // Only used for debugging/development. void tonemap_image_reinhard(image &ldr_img, const imagef &hdr_img, float exposure, bool add_noise, bool per_component, bool luma_scaling) { @@ -3678,7 +3693,7 @@ namespace basisu rand r; r.seed(128); - + for (uint32_t y = 0; y < height; y++) { for (uint32_t x = 0; x < width; x++) @@ -3713,7 +3728,7 @@ namespace basisu { //Lmapped = L / (1.0f + L); //Lmapped /= L; - + Lmapped = 1.0f / (1.0f + L); } @@ -3933,7 +3948,7 @@ namespace basisu dst_img.set_all(color_rgba(0, 0, 0, 255)); basisu::vector half_img(width * 3 * height); - + uint32_t low_h = UINT32_MAX, high_h = 0; for (uint32_t y = 0; y < height; y++) @@ -3957,7 +3972,7 @@ namespace basisu low_h = minimum(low_h, h); high_h = maximum(high_h, h); - + half_img[(x + y * width) * 3 + i] = (basist::half_float)h; } // i @@ -3974,7 +3989,7 @@ namespace basisu for (uint32_t i = 0; i < 3; i++) { basist::half_float h = half_img[(x + y * width) * 3 + i]; - + float f = (float)(h - low_h) / (float)(high_h - low_h); int iv = basisu::clamp((int)std::round(f * 255.0f), 0, 255); @@ -3988,6 +4003,328 @@ namespace basisu return true; } + bool arith_test() + { + basist::arith_fastbits_f32::init(); + + fmt_printf("random bit test\n"); + + const uint32_t N = 1000; + + // random bit test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + for (uint32_t j = 0; j < num_vals; j++) + enc.put_bit(r.bit()); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.bit(); + + uint32_t a = dec.get_bit(); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Random bit test OK\n"); + + fmt_printf("random bits test\n"); + + // random bits test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + uint32_t num_bits = r.irand(1, 20); + + for (uint32_t j = 0; j < num_vals; j++) + enc.put_bits(r.urand32() & ((1 << num_bits) - 1), num_bits); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + uint32_t num_bits = r.irand(1, 20); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.urand32() & ((1 << num_bits) - 1); + + uint32_t a = dec.get_bits(num_bits); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Random bits test OK\n"); + + fmt_printf("random adaptive bit model test\n"); + + // adaptive bit model random test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(r.bit(), bm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = r.bit(); + + uint32_t a = dec.decode_bit(bm); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + fmt_printf("Random adaptive bits test OK\n"); + + fmt_printf("random adaptive bit model 0 or 1 run test\n"); + + // adaptive bit model 0 or 1 test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(i & 1, bm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 20000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t t = i & 1; + + uint32_t a = dec.decode_bit(bm); + if (t != a) + { + fmt_printf("error!"); + return false; + } + } + } + } + + fmt_printf("Adaptive bit model 0 or 1 run test OK\n"); + + fmt_printf("random adaptive bit model 0 or 1 run 2 test\n"); + + // adaptive bit model 0 or 1 run test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 2000); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + const uint32_t run_len = r.irand(1, 128); + const uint32_t t = r.bit(); + for (uint32_t k = 0; k < run_len; k++) + enc.encode(t, bm); + } + + if (r.frand(0.0f, 1.0f) < .1f) + { + for (uint32_t q = 0; q < 1000; q++) + enc.encode(0, bm); + } + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 2000); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_bit_model bm; + bm.init(); + + for (uint32_t j = 0; j < num_vals; j++) + { + const uint32_t run_len = r.irand(1, 128); + const uint32_t t = r.bit(); + + for (uint32_t k = 0; k < run_len; k++) + { + uint32_t a = dec.decode_bit(bm); + if (a != t) + { + fmt_printf("adaptive bit model random run test failed!\n"); + return false; + } + } + } + + if (r.frand(0.0f, 1.0f) < .1f) + { + for (uint32_t q = 0; q < 1000; q++) + { + uint32_t d = dec.decode_bit(bm); + if (d != 0) + { + fmt_printf("adaptive bit model random run test failed!\n"); + return false; + } + } + } + } + } + + fmt_printf("Random data model test\n"); + + // random data model test + for (uint32_t i = 0; i < N; i++) + { + basist::arith::arith_enc enc; + enc.init(4096); + + { + basisu::rand r; + r.seed(i + 1); + const uint32_t num_vals = r.irand(1, 60000); + + uint32_t num_syms = r.irand(2, basist::arith::ArithMaxSyms); + + basist::arith::arith_data_model dm; + dm.init(num_syms); + + for (uint32_t j = 0; j < num_vals; j++) + enc.encode(r.irand(0, num_syms - 1), dm); + + enc.flush(); + } + + { + basisu::rand r; + r.seed(i + 1); + uint32_t num_vals = r.irand(1, 60000); + + const uint32_t num_syms = r.irand(2, basist::arith::ArithMaxSyms); + + basist::arith::arith_dec dec; + dec.init(enc.get_data_buf().get_ptr(), enc.get_data_buf().size()); + + basist::arith::arith_data_model dm; + dm.init(num_syms); + + for (uint32_t j = 0; j < num_vals; j++) + { + uint32_t expected = r.irand(0, num_syms - 1); + uint32_t actual = dec.decode_sym(dm); + if (actual != expected) + { + fmt_printf("adaptive data model random test failed!\n"); + return false; + } + } + } + } + + fmt_printf("Adaptive data model random test OK\n"); + + fmt_printf("Overall OK\n"); + return true; + } + static void rasterize_line(image& dst, int xs, int ys, int xe, int ye, int pred, int inc_dec, int e, int e_inc, int e_no_inc, const color_rgba& color) { int start, end, var; @@ -4023,6 +4360,7 @@ namespace basisu } } } + void draw_line(image& dst, int xs, int ys, int xe, int ye, const color_rgba& color) { if (xs > xe) @@ -4084,7 +4422,7 @@ namespace basisu int y = 0; int err = 1 - x; - while (x >= y) + while (x >= y) { dst.set_clipped(cx + x, cy + y, color); dst.set_clipped(cx + y, cy + x, color); @@ -4097,11 +4435,11 @@ namespace basisu ++y; - if (err < 0) + if (err < 0) { err += 2 * y + 1; } - else + else { --x; err += 2 * (y - x) + 1; @@ -4116,4 +4454,270 @@ namespace basisu img(x, y).a = (uint8_t)a; } + // red=3 subsets, blue=2 subsets, green=mode 6, white=mode 7, purple = 2 plane + const color_rgba g_bc7_mode_vis_colors[8] = + { + color_rgba(190, 0, 0, 255), // 0 + color_rgba(0, 0, 255, 255), // 1 + color_rgba(255, 0, 0, 255), // 2 + color_rgba(0, 0, 130, 255), // 3 + color_rgba(255, 0, 255, 255), // 4 + color_rgba(190, 0, 190, 255), // 5 + color_rgba(50, 167, 30, 255), // 6 + color_rgba(255, 255, 255, 255) // 7 + }; + + void create_bc7_debug_images( + uint32_t width, uint32_t height, + const void *pBlocks, + const char *pFilename_prefix) + { + assert(width && height && pBlocks ); + + const uint32_t num_bc7_blocks_x = (width + 3) >> 2; + const uint32_t num_bc7_blocks_y = (height + 3) >> 2; + const uint32_t total_bc7_blocks = num_bc7_blocks_x * num_bc7_blocks_y; + + image bc7_mode_vis(width, height); + + uint32_t bc7_mode_hist[9] = {}; + + uint32_t mode4_index_hist[2] = {}; + uint32_t mode4_rot_hist[4] = {}; + uint32_t mode5_rot_hist[4] = {}; + + uint32_t num_2subsets = 0, num_3subsets = 0, num_dp = 0; + + uint32_t total_solid_bc7_blocks = 0; + uint32_t num_unpack_failures = 0; + + for (uint32_t by = 0; by < num_bc7_blocks_y; by++) + { + const uint32_t base_y = by * 4; + + for (uint32_t bx = 0; bx < num_bc7_blocks_x; bx++) + { + const uint32_t base_x = bx * 4; + + const basist::bc7_block& blk = ((const basist::bc7_block *)pBlocks)[bx + by * num_bc7_blocks_x]; + + color_rgba unpacked_pixels[16]; + bool status = basist::bc7u::unpack_bc7(&blk, (basist::color_rgba*)unpacked_pixels); + if (!status) + num_unpack_failures++; + + int mode_index = basist::bc7u::determine_bc7_mode(&blk); + + bool is_solid = false; + + // assumes our transcoder's analytical BC7 encoder wrote the solid block + if (mode_index == 5) + { + const uint8_t* pBlock_bytes = (const uint8_t *)&blk; + + if (pBlock_bytes[0] == 0b00100000) + { + static const uint8_t s_tail_bytes[8] = { 0xac, 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + if ((pBlock_bytes[8] & ~3) == (s_tail_bytes[0] & ~3)) + { + if (memcmp(pBlock_bytes + 9, s_tail_bytes + 1, 7) == 0) + { + is_solid = true; + } + } + } + } + + total_solid_bc7_blocks += is_solid; + + if ((mode_index == 0) || (mode_index == 2)) + num_3subsets++; + else if ((mode_index == 1) || (mode_index == 3)) + num_2subsets++; + + bc7_mode_hist[mode_index + 1]++; + + if (mode_index == 4) + { + num_dp++; + mode4_index_hist[range_check(basist::bc7u::determine_bc7_mode_4_index_mode(&blk), 0, 1)]++; + mode4_rot_hist[range_check(basist::bc7u::determine_bc7_mode_4_or_5_rotation(&blk), 0, 3)]++; + } + else if (mode_index == 5) + { + num_dp++; + mode5_rot_hist[range_check(basist::bc7u::determine_bc7_mode_4_or_5_rotation(&blk), 0, 3)]++; + } + + color_rgba c((mode_index < 0) ? g_black_color : g_bc7_mode_vis_colors[mode_index]); + + if (is_solid) + c.set(64, 0, 64, 255); + + bc7_mode_vis.fill_box(base_x, base_y, 4, 4, c); + + } // bx + + } // by + + fmt_debug_printf("--------- BC7 statistics:\n"); + fmt_debug_printf("\nTotal BC7 unpack failures: {}\n", num_unpack_failures); + fmt_debug_printf("Total solid blocks: {} {3.2}%\n", total_solid_bc7_blocks, (float)total_solid_bc7_blocks * (float)100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nTotal 2-subsets: {} {3.2}%\n", num_2subsets, (float)num_2subsets * 100.0f / (float)total_bc7_blocks); + fmt_debug_printf("Total 3-subsets: {} {3.2}%\n", num_3subsets, (float)num_3subsets * 100.0f / (float)total_bc7_blocks); + fmt_debug_printf("Total Dual Plane: {} {3.2}%\n", num_dp, (float)num_dp * 100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nBC7 mode histogram:\n"); + for (int i = -1; i <= 7; i++) + { + fmt_debug_printf(" {}: {} {3.3}%\n", i, bc7_mode_hist[1 + i], (float)bc7_mode_hist[1 + i] * 100.0f / (float)total_bc7_blocks); + } + + fmt_debug_printf("\nMode 4 index bit histogram: {} {3.2}%, {} {3.2}%\n", + mode4_index_hist[0], (float)mode4_index_hist[0] * 100.0f / (float)total_bc7_blocks, + mode4_index_hist[1], (float)mode4_index_hist[1] * 100.0f / (float)total_bc7_blocks); + + fmt_debug_printf("\nMode 4 rotation histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + { + fmt_debug_printf(" {}: {} {3.2}%\n", i, mode4_rot_hist[i], (float)mode4_rot_hist[i] * 100.0f / (float)total_bc7_blocks); + } + + fmt_debug_printf("\nMode 5 rotation histogram:\n"); + for (uint32_t i = 0; i < 4; i++) + { + fmt_debug_printf(" {}: {} {3.2}%\n", i, mode5_rot_hist[i], (float)mode5_rot_hist[i] * 100.0f / (float)total_bc7_blocks); + } + + if (pFilename_prefix) + { + std::string mode_vis_filename(std::string(pFilename_prefix) + "bc7_mode_vis.png"); + save_png(mode_vis_filename, bc7_mode_vis); + + fmt_debug_printf("Wrote BC7 mode visualization to PNG file {}\n", mode_vis_filename); + } + + fmt_debug_printf("--------- End BC7 statistics\n"); + fmt_debug_printf("\n"); + } + + static inline float edge(const vec2F& a, const vec2F& b, const vec2F& pos) + { + return (pos[0] - a[0]) * (b[1] - a[1]) - (pos[1] - a[1]) * (b[0] - a[0]); + } + + void draw_tri2(image& dst, const image* pTex, const tri2& tri, bool alpha_blend) + { + assert(dst.get_total_pixels()); + + float area = edge(tri.p0, tri.p1, tri.p2); + if (std::fabs(area) < 1e-6f) + return; + + const float oo_area = 1.0f / area; + + int minx = (int)std::floor(basisu::minimum(tri.p0[0], tri.p1[0], tri.p2[0] )); + int miny = (int)std::floor(basisu::minimum(tri.p0[1], tri.p1[1], tri.p2[1] )); + + int maxx = (int)std::ceil(basisu::maximum(tri.p0[0], tri.p1[0], tri.p2[0])); + int maxy = (int)std::ceil(basisu::maximum(tri.p0[1], tri.p1[1], tri.p2[1])); + + auto clamp8 = [&](float fv) { int v = (int)(fv + .5f); if (v < 0) v = 0; else if (v > 255) v = 255; return (uint8_t)v; }; + + if ((maxx < 0) || (maxy < 0)) + return; + if ((minx >= (int)dst.get_width()) || (miny >= (int)dst.get_height())) + return; + + if (minx < 0) + minx = 0; + if (maxx >= (int)dst.get_width()) + maxx = dst.get_width() - 1; + if (miny < 0) + miny = 0; + if (maxy >= (int)dst.get_height()) + maxy = dst.get_height() - 1; + + vec4F tex(1.0f); + + for (int y = miny; y <= maxy; ++y) + { + assert((y >= 0) && (y < (int)dst.get_height())); + + for (int x = minx; x <= maxx; ++x) + { + assert((x >= 0) && (x < (int)dst.get_width())); + + vec2F p{ (float)x + 0.5f, (float)y + 0.5f }; + + float w0 = edge(tri.p1, tri.p2, p) * oo_area; + float w1 = edge(tri.p2, tri.p0, p) * oo_area; + float w2 = edge(tri.p0, tri.p1, p) * oo_area; + + if ((w0 < 0) || (w1 < 0) || (w2 < 0)) + continue; + + float u = tri.t0[0] * w0 + tri.t1[0] * w1 + tri.t2[0] * w2; + float v = tri.t0[1] * w0 + tri.t1[1] * w1 + tri.t2[1] * w2; + + if (pTex) + tex = pTex->get_filtered_vec4F(u * float(pTex->get_width()), v * float(pTex->get_height())) * (1.0f / 255.0f); + + float r = (float)tri.c0.r * w0 + (float)tri.c1.r * w1 + (float)tri.c2.r * w2; + float g = (float)tri.c0.g * w0 + (float)tri.c1.g * w1 + (float)tri.c2.g * w2; + float b = (float)tri.c0.b * w0 + (float)tri.c1.b * w1 + (float)tri.c2.b * w2; + float a = (float)tri.c0.a * w0 + (float)tri.c1.a * w1 + (float)tri.c2.a * w2; + + r *= tex[0]; + g *= tex[1]; + b *= tex[2]; + a *= tex[3]; + + if (alpha_blend) + { + color_rgba dst_color(dst(x, y)); + + const float fa = (float)a * (1.0f / 255.0f); + + r = lerp((float)dst_color[0], r, fa); + g = lerp((float)dst_color[1], g, fa); + b = lerp((float)dst_color[2], b, fa); + a = lerp((float)dst_color[3], a, fa); + + dst(x, y) = color_rgba(clamp8(r), clamp8(g), clamp8(b), clamp8(a)); + } + else + { + dst(x, y) = color_rgba(clamp8(r), clamp8(g), clamp8(b), clamp8(a)); + } + + } // x + } // y + } + + // macro sent by CMakeLists.txt file when (TARGET_WASM AND WASM_THREADING) +#if BASISU_WASI_THREADS + // Default to 8 - seems reasonable. + static int g_num_wasi_threads = 8; +#else + static int g_num_wasi_threads = 0; +#endif + + void set_num_wasi_threads(uint32_t num_threads) + { + g_num_wasi_threads = num_threads; + } + + int get_num_hardware_threads() + { +#ifdef __wasi__ + int num_threads = g_num_wasi_threads; +#else + int num_threads = std::thread::hardware_concurrency(); +#endif + + return num_threads; + } + } // namespace basisu diff --git a/encoder/basisu_enc.h b/encoder/basisu_enc.h index 0c644e8..d0b7741 100644 --- a/encoder/basisu_enc.h +++ b/encoder/basisu_enc.h @@ -1,5 +1,5 @@ // basisu_enc.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ namespace basisu { extern uint8_t g_hamming_dist[256]; extern const uint8_t g_debug_font8x8_basic[127 - 32 + 1][8]; + extern float g_srgb_to_linear_table[256]; // sRGB EOTF->linear light [0,1], 1=~100 nits // true if basisu_encoder_init() has been called and returned. extern bool g_library_initialized; @@ -61,7 +62,7 @@ namespace basisu void error_vprintf(const char* pFmt, va_list args); void error_printf(const char *pFmt, ...); - + template inline void fmt_error_printf(const char* pFmt, Args&&... args) { @@ -72,7 +73,7 @@ namespace basisu } void platform_sleep(uint32_t ms); - + // Helpers inline uint8_t clamp255(int32_t i) @@ -92,21 +93,27 @@ namespace basisu return val << shift; } - inline int32_t clampi(int32_t value, int32_t low, int32_t high) - { - if (value < low) - value = low; - else if (value > high) - value = high; - return value; + inline int32_t clampi(int32_t value, int32_t low, int32_t high) + { + if (value < low) + value = low; + else if (value > high) + value = high; + return value; } inline uint8_t mul_8(uint32_t v, uint32_t a) { - v = v * a + 128; + v = v * a + 128; return (uint8_t)((v + (v >> 8)) >> 8); } + inline int fast_roundf_pos_int(float x) + { + assert(x >= 0.0f); + return (int)(x + 0.5f); + } + inline int fast_roundf_int(float x) { return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); @@ -163,7 +170,7 @@ namespace basisu return bits; } - + // Open interval inline int bounds_check(int v, int l, int h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } inline uint32_t bounds_check(uint32_t v, uint32_t l, uint32_t h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } @@ -211,10 +218,10 @@ namespace basisu return -1; return (int)res; } - + // Hashing - - inline uint32_t bitmix32c(uint32_t v) + + inline uint32_t bitmix32c(uint32_t v) { v = (v + 0x7ed55d16) + (v << 12); v = (v ^ 0xc761c23c) ^ (v >> 19); @@ -225,7 +232,7 @@ namespace basisu return v; } - inline uint32_t bitmix32(uint32_t v) + inline uint32_t bitmix32(uint32_t v) { v -= (v << 6); v ^= (v >> 17); @@ -246,29 +253,7 @@ namespace basisu seed = seed ^ (seed >> 15); return seed; } - - uint32_t hash_hsieh(const uint8_t* pBuf, size_t len); - - template - struct bit_hasher - { - inline std::size_t operator()(const Key& k) const - { - return hash_hsieh(reinterpret_cast(&k), sizeof(k)); - } - }; - - struct string_hasher - { - inline std::size_t operator()(const std::string& k) const - { - size_t l = k.size(); - if (!l) - return 0; - return hash_hsieh(reinterpret_cast(k.c_str()), l); - } - }; - + class running_stat { public: @@ -351,7 +336,7 @@ namespace basisu }; // Linear algebra - + template class vec { @@ -456,7 +441,7 @@ namespace basisu inline const T *get_ptr() const { return reinterpret_cast(&m_v[0]); } inline T *get_ptr() { return reinterpret_cast(&m_v[0]); } - + inline vec operator- () const { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = -m_v[i]; return res; } inline vec operator+ () const { return *this; } inline vec &operator+= (const vec &other) { for (uint32_t i = 0; i < N; i++) m_v[i] += other.m_v[i]; return *this; } @@ -465,17 +450,19 @@ namespace basisu inline vec &operator*=(const vec &other) { for (uint32_t i = 0; i < N; i++) m_v[i] *= other.m_v[i]; return *this; } inline vec &operator/= (T s) { for (uint32_t i = 0; i < N; i++) m_v[i] /= s; return *this; } inline vec &operator*= (T s) { for (uint32_t i = 0; i < N; i++) m_v[i] *= s; return *this; } - + friend inline vec operator+(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] + rhs.m_v[i]; return res; } friend inline vec operator-(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] - rhs.m_v[i]; return res; } friend inline vec operator*(const vec &lhs, T val) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] * val; return res; } friend inline vec operator*(T val, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = val * rhs.m_v[i]; return res; } friend inline vec operator/(const vec &lhs, T val) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] / val; return res; } friend inline vec operator/(const vec &lhs, const vec &rhs) { vec res; for (uint32_t i = 0; i < N; i++) res.m_v[i] = lhs.m_v[i] / rhs.m_v[i]; return res; } - + static inline T dot_product(const vec &lhs, const vec &rhs) { T res = lhs.m_v[0] * rhs.m_v[0]; for (uint32_t i = 1; i < N; i++) res += lhs.m_v[i] * rhs.m_v[i]; return res; } + static inline T dot_product3(const vec& lhs, const vec& rhs) { T res = lhs.m_v[0] * rhs.m_v[0]; for (uint32_t i = 1; i < minimum(3u, N); i++) res += lhs.m_v[i] * rhs.m_v[i]; return res; } inline T dot(const vec &rhs) const { return dot_product(*this, rhs); } + inline T dot3(const vec& rhs) const { return dot_product3(*this, rhs); } inline T norm() const { return dot_product(*this, *this); } inline T length() const { return sqrt(norm()); } @@ -546,7 +533,7 @@ namespace basisu template struct bitwise_copyable< vec > { enum { cFlag = true }; }; template struct bitwise_movable< vec > { enum { cFlag = true }; }; - + template class matrix { @@ -810,7 +797,7 @@ namespace basisu } #undef BASISU_GET_KEY - + // Very simple job pool with no dependencies. class job_pool { @@ -820,24 +807,24 @@ namespace basisu // num_threads is the TOTAL number of job pool threads, including the calling thread! So 2=1 new thread, 3=2 new threads, etc. job_pool(uint32_t num_threads); ~job_pool(); - + void add_job(const std::function& job); void add_job(std::function&& job); void wait_for_all(); size_t get_total_threads() const { return 1 + m_threads.size(); } - + private: std::vector m_threads; std::vector > m_queue; - + std::mutex m_mutex; std::condition_variable m_has_work; std::condition_variable m_no_more_jobs; - + uint32_t m_num_active_jobs; - + std::atomic m_kill_flag; std::atomic m_num_active_workers; @@ -882,7 +869,7 @@ namespace basisu return *this; } }; - + class color_rgba { public: @@ -1006,7 +993,7 @@ namespace basisu inline const uint8_t &operator[] (uint32_t index) const { assert(index < 4); return m_comps[index]; } inline uint8_t &operator[] (uint32_t index) { assert(index < 4); return m_comps[index]; } - + inline void clear() { m_comps[0] = 0; @@ -1042,7 +1029,7 @@ namespace basisu } inline int get_601_luma() const { return (19595U * m_comps[0] + 38470U * m_comps[1] + 7471U * m_comps[2] + 32768U) >> 16U; } - inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } + inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } inline int get_luma(bool luma_601) const { return luma_601 ? get_601_luma() : get_709_luma(); } inline uint32_t get_bgra_uint32() const { return b | (g << 8) | (r << 16) | (a << 24); } @@ -1114,7 +1101,7 @@ namespace basisu return color_distance(e1, e2, alpha); } } - + inline uint32_t color_distance(bool perceptual, const color_rgba &e1, const color_rgba &e2, bool alpha) { if (perceptual) @@ -1127,7 +1114,7 @@ namespace basisu int delta_l = dr * 14 + dg * 45 + db * 5; int delta_cr = dr * 64 - delta_l; int delta_cb = db * 64 - delta_l; - + // not >> 6, so the output is scaled by 7 bits, not 6 (to match the original function which scaled by 7, but had rare overflow issues) uint32_t id = ((uint32_t)(delta_l * delta_l) >> 5U) + ((((uint32_t)(delta_cr * delta_cr) >> 5U) * 26U) >> 7U) + @@ -1151,7 +1138,7 @@ namespace basisu if (alpha) { int da = (e1.a - e2.a) << 7; - + // This shouldn't overflow if da is 255 or -255: 29.99 bits after squaring. uint32_t ea = ((uint32_t)(da * da) >> 7U); id += ea; @@ -1160,7 +1147,7 @@ namespace basisu // Make sure it can't overflow assert((((int64_t)da * (int64_t)da) >> 7) == ea); #endif - + } return id; @@ -1222,7 +1209,7 @@ namespace basisu return true; } - + inline std::string string_tolower(const std::string& s) { std::string result(s); @@ -1267,7 +1254,7 @@ namespace basisu char fname_buf[_MAX_FNAME] = { 0 }; char ext_buf[_MAX_EXT] = { 0 }; - errno_t error = _splitpath_s(p, + errno_t error = _splitpath_s(p, pDrive ? drive_buf : NULL, pDrive ? _MAX_DRIVE : 0, pDir ? dir_buf : NULL, pDir ? _MAX_DIR : 0, pFilename ? fname_buf : NULL, pFilename ? _MAX_FNAME : 0, @@ -1299,7 +1286,7 @@ namespace basisu if ((pDir->size()) && (pDir->back() != '/')) *pDir += "/"; } - + if (pFilename) { *pFilename = pBaseName; @@ -1326,7 +1313,7 @@ namespace basisu return (c == '/'); #endif } - + inline bool is_drive_separator(char c) { #ifdef _WIN32 @@ -1354,7 +1341,7 @@ namespace basisu string_combine_path(dst, p, q); string_combine_path(dst, dst.c_str(), r); } - + inline void string_combine_path_and_extension(std::string &dst, const char *p, const char *q, const char *r, const char *pExt) { string_combine_path(dst, p, q, r); @@ -1556,7 +1543,7 @@ namespace basisu codebook.resize(0); codebook.reserve(max_clusters); - + uint32_t node_index = 0; while (true) @@ -1567,7 +1554,7 @@ namespace basisu { codebook.resize(codebook.size() + 1); codebook.back() = cur.m_training_vecs; - + if (node_stack.empty()) break; @@ -1575,7 +1562,7 @@ namespace basisu node_stack.pop_back(); continue; } - + node_stack.push_back(cur.m_right_index); node_index = cur.m_left_index; } @@ -1616,7 +1603,7 @@ namespace basisu assert(node.is_leaf()); var_heap.delete_top(); - + if (node.m_training_vecs.size() > 1) { if (split_node(node_index, var_heap, l_children, r_children)) @@ -1705,7 +1692,7 @@ namespace basisu m_nodes[node_index].m_left_index = l_child_index; m_nodes[node_index].m_right_index = r_child_index; - + m_nodes[node_index].m_codebook_index = m_next_codebook_index; m_next_codebook_index++; @@ -1719,7 +1706,7 @@ namespace basisu if ((l_child.m_var <= 0.0f) && (l_child.m_training_vecs.size() > 1)) { TrainingVectorType v(m_training_vecs[l_child.m_training_vecs[0]].first); - + for (uint32_t i = 1; i < l_child.m_training_vecs.size(); i++) { if (!(v == m_training_vecs[l_child.m_training_vecs[i]].first)) @@ -1746,10 +1733,10 @@ namespace basisu if ((l_child.m_var > 0.0f) && (l_child.m_training_vecs.size() > 1)) var_heap.add_heap(l_child_index, l_child.m_var); - + if ((r_child.m_var > 0.0f) && (r_child.m_training_vecs.size() > 1)) var_heap.add_heap(r_child_index, r_child.m_var); - + return true; } @@ -1845,7 +1832,7 @@ namespace basisu for (uint32_t i = 0; i < node.m_training_vecs.size(); i++) { const TrainingVectorType& v = m_training_vecs[node.m_training_vecs[i]].first; - + l = TrainingVectorType::component_min(l, v); h = TrainingVectorType::component_max(h, v); } @@ -1926,8 +1913,8 @@ namespace basisu const uint32_t cMaxIters = 6; for (uint32_t iter = 0; iter < cMaxIters; iter++) { - l_children.resize(0); - r_children.resize(0); + l_children.resize(0); + r_children.resize(0); TrainingVectorType new_l_child(cZero), new_r_child(cZero); @@ -1975,11 +1962,12 @@ namespace basisu r_weight = 0; TrainingVectorType firstVec; + firstVec.clear(); for (uint32_t i = 0; i < node.m_training_vecs.size(); i++) { const TrainingVectorType& v = m_training_vecs[node.m_training_vecs[i]].first; const uint64_t weight = m_training_vecs[node.m_training_vecs[i]].second; - + if ((!i) || (v == firstVec)) { firstVec = v; @@ -2081,7 +2069,7 @@ namespace basisu } Quantizer quantizers[cMaxThreads]; - + bool success_flags[cMaxThreads]; clear_obj(success_flags); @@ -2180,12 +2168,12 @@ namespace basisu // rg 6/24/2025 - Cross platform determinism #if 0 - typedef std::unordered_map < typename Quantizer::training_vec_type, weighted_block_group, + typedef std::unordered_map < typename Quantizer::training_vec_type, weighted_block_group, training_vec_bit_hasher> group_hash; #else typedef std::map< typename Quantizer::training_vec_type, weighted_block_group > group_hash; #endif - + //interval_timer tm; //tm.start(); @@ -2197,7 +2185,7 @@ namespace basisu #endif weighted_block_group g; - + if (even_odd_input_pairs_equal) { g.m_indices.resize(2); @@ -2282,7 +2270,7 @@ namespace basisu typename group_hash::const_iterator group_iter = unique_vec_iters[group_index]; const uint_vec& training_vec_indices = group_iter->second.m_indices; - + append_vector(codebook.back(), training_vec_indices); } } @@ -2359,7 +2347,7 @@ namespace basisu const double inv_total = 1.0f / total; const double neg_inv_log2 = -1.0f / log(2.0f); - + double e = 0.0f; for (uint32_t i = 0; i < m_hist.size(); i++) if (m_hist[i]) @@ -2368,7 +2356,7 @@ namespace basisu return e; } }; - + struct sym_freq { uint32_t m_key; @@ -2378,7 +2366,7 @@ namespace basisu sym_freq *canonical_huffman_radix_sort_syms(uint32_t num_syms, sym_freq *pSyms0, sym_freq *pSyms1); void canonical_huffman_calculate_minimum_redundancy(sym_freq *A, int num_syms); void canonical_huffman_enforce_max_code_size(int *pNum_codes, int code_list_len, int max_code_size); - + class huffman_encoding_table { public: @@ -2399,7 +2387,7 @@ namespace basisu bool init(uint32_t num_syms, const uint16_t *pFreq, uint32_t max_code_size); bool init(uint32_t num_syms, const uint32_t *pSym_freq, uint32_t max_code_size); - + inline const uint16_vec &get_codes() const { return m_codes; } inline const uint8_vec &get_code_sizes() const { return m_code_sizes; } @@ -2430,7 +2418,7 @@ namespace basisu m_bytes(other.m_bytes), m_bit_buffer(other.m_bit_buffer), m_bit_buffer_size(other.m_bit_buffer_size), - m_total_bits(other.m_total_bits) + m_total_bits(other.m_total_bits) { } @@ -2512,7 +2500,7 @@ namespace basisu m_bit_buffer = 0; m_bit_buffer_size = 0; - + return 8; } @@ -2561,7 +2549,7 @@ namespace basisu if (v < u) return put_bits(v, k); - + uint32_t x = v + u; assert((x >> 1) >= u); @@ -2573,20 +2561,20 @@ namespace basisu inline uint32_t put_rice(uint32_t v, uint32_t m) { assert(m); - + const uint64_t start_bits = m_total_bits; uint32_t q = v >> m, r = v & ((1 << m) - 1); // rice coding sanity check assert(q <= 64); - + for (; q > 16; q -= 16) put_bits(0xFFFF, 16); put_bits((1 << q) - 1, q); put_bits(r << 1, m + 1); - + return (uint32_t)(m_total_bits - start_bits); } @@ -2596,13 +2584,13 @@ namespace basisu const uint32_t chunk_size = 1 << chunk_bits; const uint32_t chunk_mask = chunk_size - 1; - + uint32_t total_bits = 0; for ( ; ; ) { uint32_t next_v = v >> chunk_bits; - + total_bits += put_bits((v & chunk_mask) | (next_v ? chunk_size : 0), chunk_bits + 1); if (!next_v) break; @@ -2619,11 +2607,11 @@ namespace basisu { for (uint32_t i = 0; i < other.m_bytes.size(); i++) put_bits(other.m_bytes[i], 8); - + if (other.m_bit_buffer_size) put_bits(other.m_bit_buffer, other.m_bit_buffer_size); } - + private: uint8_vec m_bytes; uint32_t m_bit_buffer, m_bit_buffer_size; @@ -2653,7 +2641,7 @@ namespace basisu inline void init(uint32_t bits_per_sym, uint32_t total_syms_per_group) { assert((bits_per_sym * total_syms_per_group) <= 16 && total_syms_per_group >= 1 && bits_per_sym >= 1); - + m_bits_per_sym = bits_per_sym; m_total_syms_per_group = total_syms_per_group; m_cur_sym_bits = 0; @@ -2707,7 +2695,7 @@ namespace basisu return true; } - + inline uint32_t emit_next_sym(bitwise_coder &c) { uint32_t bits = 0; @@ -2737,7 +2725,7 @@ namespace basisu bool huffman_test(int rand_seed); // VQ index reordering - + class palette_index_reorderer { public: @@ -2758,7 +2746,7 @@ namespace basisu typedef float(*pEntry_dist_func)(uint32_t i, uint32_t j, void *pCtx); void init(uint32_t num_indices, const uint32_t *pIndices, uint32_t num_syms, pEntry_dist_func pDist_func, void *pCtx, float dist_func_weight); - + // Table remaps old to new symbol indices inline const uint_vec &get_remap_table() const { return m_remap_table; } @@ -2779,12 +2767,12 @@ namespace basisu class image { public: - image() : + image() : m_width(0), m_height(0), m_pitch(0) { } - image(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : + image(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : m_width(0), m_height(0), m_pitch(0) { resize(w, h, p); @@ -2850,7 +2838,7 @@ namespace basisu image &clear() { - m_width = 0; + m_width = 0; m_height = 0; m_pitch = 0; clear_vector(m_pixels); @@ -2878,7 +2866,7 @@ namespace basisu void init(const uint8_t *pImage, uint32_t width, uint32_t height, uint32_t comps) { assert(comps >= 1 && comps <= 4); - + resize(width, height); for (uint32_t y = 0; y < height; y++) @@ -2970,7 +2958,7 @@ namespace basisu p = w; clear(); - + if ((!p) || (!w) || (!h)) return *this; @@ -3049,8 +3037,8 @@ namespace basisu y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); return m_pixels[x + y * m_pitch]; } - - inline image &set_clipped(int x, int y, const color_rgba &c) + + inline image &set_clipped(int x, int y, const color_rgba &c) { if ((static_cast(x) < m_width) && (static_cast(y) < m_height)) (*this)(x, y) = c; @@ -3214,7 +3202,7 @@ namespace basisu } void debug_text(uint32_t x_ofs, uint32_t y_ofs, uint32_t x_scale, uint32_t y_scale, const color_rgba &fg, const color_rgba *pBG, bool alpha_only, const char* p, ...); - + // bilinear filtering vec4F get_filtered_vec4F(float x, float y) const { @@ -3257,7 +3245,7 @@ namespace basisu return result; } - + private: uint32_t m_width, m_height, m_pitch; // all in pixels color_rgba_vec m_pixels; @@ -3269,7 +3257,7 @@ namespace basisu inline bool is_solid_block(uint32_t n, const color_rgba* pPixels) { assert(n); - + if (n <= 1) return true; @@ -3300,12 +3288,12 @@ namespace basisu class imagef { public: - imagef() : + imagef() : m_width(0), m_height(0), m_pitch(0) { } - imagef(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : + imagef(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX) : m_width(0), m_height(0), m_pitch(0) { resize(w, h, p); @@ -3365,7 +3353,7 @@ namespace basisu imagef &clear() { - m_width = 0; + m_width = 0; m_height = 0; m_pitch = 0; clear_vector(m_pixels); @@ -3421,7 +3409,7 @@ namespace basisu set_clipped(x + ix, y + iy, c); return *this; } - + imagef &crop(uint32_t w, uint32_t h, uint32_t p = UINT32_MAX, const vec4F &background = vec4F(0,0,0,1)) { if (p == UINT32_MAX) @@ -3440,7 +3428,7 @@ namespace basisu cur_state.swap(m_pixels); m_pixels.resize(p * h); - + for (uint32_t y = 0; y < h; y++) { for (uint32_t x = 0; x < w; x++) @@ -3503,8 +3491,8 @@ namespace basisu y = wrap_v ? posmod(y, m_height) : clamp(y, 0, m_height - 1); return m_pixels[x + y * m_pitch]; } - - inline imagef &set_clipped(int x, int y, const vec4F &c) + + inline imagef &set_clipped(int x, int y, const vec4F &c) { if ((static_cast(x) < m_width) && (static_cast(y) < m_height)) (*this)(x, y) = c; @@ -3589,7 +3577,7 @@ namespace basisu { float &p = c[s]; union { float f; uint32_t u; } x; x.f = p; - + if ((std::isnan(p)) || (std::isinf(p)) || (x.u == 0x80000000)) { if (std::isnan(p)) @@ -3634,14 +3622,14 @@ namespace basisu fprintf(stderr, "One or more input pixels was negative -- setting these pixel components to 0 because ASTC HDR doesn't support signed values.\n"); neg_msg = true; } - + status = false; } if (p > highest_mag) { p = highest_mag; - + if (!clamp_msg) { fprintf(stderr, "One or more input pixels had to be clamped to %f.\n", highest_mag); @@ -3705,7 +3693,7 @@ namespace basisu return result; } - + private: uint32_t m_width, m_height, m_pitch; // all in pixels vec4F_vec m_pixels; @@ -3767,15 +3755,17 @@ namespace basisu }; extern fast_linear_to_srgb g_fast_linear_to_srgb; - + // Image metrics - + class image_metrics { public: // TODO: Add ssim + uint32_t m_width, m_height; double m_max, m_mean, m_mean_squared, m_rms, m_psnr, m_ssim; bool m_has_neg, m_hf_mag_overflow, m_any_abnormal; + uint64_t m_sum_a, m_sum_b; image_metrics() { @@ -3784,6 +3774,8 @@ namespace basisu void clear() { + m_width = 0; + m_height = 0; m_max = 0; m_mean = 0; m_mean_squared = 0; @@ -3793,10 +3785,23 @@ namespace basisu m_has_neg = false; m_hf_mag_overflow = false; m_any_abnormal = false; + m_sum_a = 0; + m_sum_b = 0; } - void print(const char *pPrefix = nullptr) { printf("%sMax: %3.3f Mean: %3.3f RMS: %3.3f PSNR: %2.3f dB\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr); } - void print_hp(const char* pPrefix = nullptr) { printf("%sMax: %3.6f Mean: %3.6f RMS: %3.6f PSNR: %2.6f dB, Any Neg: %u, Half float overflow: %u, Any NaN/Inf: %u\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal); } + void print(const char *pPrefix = nullptr) + { + //fmt_printf("{}Max: {3.3} Mean: {3.3} RMS: {3.3} PSNR: {2.3} dB, Sums: {} {}, Dim: {}x{}\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_sum_a, m_sum_b, m_width, m_height); + fmt_printf("{}Max: {3.3} Mean: {3.3} RMS: {3.3} PSNR: {2.3} dB\n", pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr); + } + + void print_hp(const char* pPrefix = nullptr) + { + //fmt_printf("{}Max: {3.6} Mean: {3.6} RMS: {3.6} PSNR: {2.6} dB, Any Neg: {}, Half float overflow: {}, Any NaN/Inf: {}, Sums: {} {}, Dim: {}x{}\n", + // pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal, m_sum_a, m_sum_b, m_width, m_height); + fmt_printf("{}Max: {3.6} Mean: {3.6} RMS: {3.6} PSNR: {2.6} dB, Any Neg: {}, Half float overflow: {}, Any NaN/Inf: {}\n", + pPrefix ? pPrefix : "", m_max, m_mean, m_rms, m_psnr, m_has_neg, m_hf_mag_overflow, m_any_abnormal); + } void calc(const imagef& a, const imagef& b, uint32_t first_chan = 0, uint32_t total_chans = 0, bool avg_comp_error = true, bool log = false); void calc_half(const imagef& a, const imagef& b, uint32_t first_chan, uint32_t total_chans, bool avg_comp_error); @@ -3820,7 +3825,7 @@ namespace basisu bool load_jpg(const char *pFilename, image& img); bool load_jpg(const uint8_t* pBuf, size_t buf_size, image& img); inline bool load_jpg(const std::string &filename, image &img) { return load_jpg(filename.c_str(), img); } - + // Currently loads .PNG, .TGA, or .JPG bool load_image(const char* pFilename, image& img); inline bool load_image(const std::string &filename, image &img) { return load_image(filename.c_str(), img); } @@ -3829,9 +3834,9 @@ namespace basisu // Supports .HDR and most (but not all) .EXR's (see TinyEXR). bool load_image_hdr(const char* pFilename, imagef& img, bool ldr_srgb_to_linear = true, float linear_nit_multiplier = 1.0f, float ldr_black_bias = 0.0f); - + inline bool load_image_hdr(const std::string& filename, imagef& img, bool ldr_srgb_to_linear = true, float linear_nit_multiplier = 1.0f, float ldr_black_bias = 0.0f) - { + { return load_image_hdr(filename.c_str(), img, ldr_srgb_to_linear, linear_nit_multiplier, ldr_black_bias); } @@ -3849,7 +3854,7 @@ namespace basisu uint8_t *read_tga(const uint8_t *pBuf, uint32_t buf_size, int &width, int &height, int &n_chans); uint8_t *read_tga(const char *pFilename, int &width, int &height, int &n_chans); - + struct rgbe_header_info { std::string m_program; @@ -3861,13 +3866,13 @@ namespace basisu double m_exposure; // watts/steradian/m^2. bool m_has_exposure; - void clear() - { - m_program.clear(); - m_gamma = 1.0f; - m_has_gamma = false; - m_exposure = 1.0f; - m_has_exposure = false; + void clear() + { + m_program.clear(); + m_gamma = 1.0f; + m_has_gamma = false; + m_exposure = 1.0f; + m_has_exposure = false; } }; @@ -3879,7 +3884,7 @@ namespace basisu bool read_exr(const char* pFilename, imagef& img, int& n_chans); bool read_exr(const void* pMem, size_t mem_size, imagef& img); - + enum { WRITE_EXR_LINEAR_HINT = 1, // hint for lossy comp. methods: exr_perceptual_treatment_t, logarithmic or linear, defaults to logarithmic @@ -3889,7 +3894,7 @@ namespace basisu // Supports 1 (Y), 3 (RGB), or 4 (RGBA) channel images. bool write_exr(const char* pFilename, const imagef& img, uint32_t n_chans, uint32_t flags); - + enum { cImageSaveGrayscale = 1, @@ -3899,25 +3904,28 @@ namespace basisu bool save_png(const char* pFilename, const image& img, uint32_t image_save_flags = 0, uint32_t grayscale_comp = 0); inline bool save_png(const std::string &filename, const image &img, uint32_t image_save_flags = 0, uint32_t grayscale_comp = 0) { return save_png(filename.c_str(), img, image_save_flags, grayscale_comp); } + bool save_qoi(const char* pFilename, const image& img, uint32_t qoi_colorspace = 0); + inline bool save_qoi(const std::string& filename, const image& img, uint32_t qoi_colorspace = 0) { return save_qoi(filename.c_str(), img, qoi_colorspace); } + bool read_file_to_vec(const char* pFilename, uint8_vec& data); - bool read_file_to_data(const char* pFilename, void *pData, size_t len); + bool read_file_to_data(const char* pFilename, void *pData, size_t len); bool write_data_to_file(const char* pFilename, const void* pData, size_t len); - + inline bool write_vec_to_file(const char* pFilename, const uint8_vec& v) { return v.size() ? write_data_to_file(pFilename, &v[0], v.size()) : write_data_to_file(pFilename, "", 0); } - + bool image_resample(const image &src, image &dst, bool srgb = false, - const char *pFilter = "lanczos4", float filter_scale = 1.0f, + const char *pFilter = "lanczos4", float filter_scale = 1.0f, bool wrapping = false, - uint32_t first_comp = 0, uint32_t num_comps = 4); + uint32_t first_comp = 0, uint32_t num_comps = 4, float filter_scale_y = -1.0f); - bool image_resample(const imagef& src, imagef& dst, + bool image_resample(const imagef& src, imagef& dst, const char* pFilter = "lanczos4", float filter_scale = 1.0f, bool wrapping = false, uint32_t first_comp = 0, uint32_t num_comps = 4); - + // Timing - + typedef uint64_t timer_ticks; class interval_timer @@ -3930,7 +3938,7 @@ namespace basisu double get_elapsed_secs() const; inline double get_elapsed_ms() const { return 1000.0f* get_elapsed_secs(); } - + static void init(); static inline timer_ticks get_ticks_per_sec() { return g_freq; } static timer_ticks get_ticks(); @@ -4006,7 +4014,7 @@ namespace basisu void tonemap_image_reinhard(image& ldr_img, const imagef& hdr_img, float exposure, bool add_noise = false, bool per_component = true, bool luma_scaling = false); bool tonemap_image_compressive(image& dst_img, const imagef& hdr_test_img); bool tonemap_image_compressive2(image& dst_img, const imagef& hdr_test_img); - + // Intersection enum eClear { cClear = 0 }; enum eInitExpand { cInitExpand = 0 }; @@ -4297,9 +4305,9 @@ namespace basisu BASISU_FORCE_INLINE float fast_half_to_float_pos_not_inf_or_nan(basist::half_float h) { assert(!basist::half_is_signed(h) && !basist::is_half_inf_or_nan(h)); - + // add 112 to the exponent (112+half float's exp bias of 15=float32's bias of 127) - static const fu32 K = { 0x77800000 }; + static const fu32 K = { 0x77800000 }; fu32 o; o.u = h << 13; @@ -4315,7 +4323,7 @@ namespace basisu // Sutract 112 from the exponent, to change the bias from 127 to 15. static const fu32 g_f_to_h{ 0x7800000 }; - + fu32 fu; fu.f = minimum((float)basist::MAX_HALF_FLOAT, fabsf(f)) * g_f_to_h.f; @@ -4327,17 +4335,17 @@ namespace basisu { assert(!isnan(f) && !isinf(f)); assert((f >= 0.0f) && (f <= basist::MAX_HALF_FLOAT)); - + // Sutract 112 from the exponent, to change the bias from 127 to 15. static const fu32 g_f_to_h{ 0x7800000 }; fu32 fu; fu.f = f * g_f_to_h.f; - + return (basist::half_float)((fu.u >> (23 - 10)) & 0x7FFF); } - + inline basist::half_float fast_float_to_half_no_clamp_neg_nan_or_inf(float f) { assert(!isnan(f) && !isinf(f)); @@ -4363,6 +4371,28 @@ namespace basisu return (basist::half_float)h; } + bool arith_test(); + + void set_image_alpha(image& img, uint32_t a); + + void create_bc7_debug_images( + uint32_t width, uint32_t height, + const void* pBlocks, + const char* pFilename_prefix); + + struct tri2 + { + vec2F p0, p1, p2; + vec2F t0, t1, t2; + color_rgba c0, c1, c2; + }; + + // simple non-perspective correct triangle rasterizer with texture mapping, useful for generating randomized test data + void draw_tri2(image& dst, const image* pTex, const tri2& tri, bool alpha_blend); + + void set_num_wasi_threads(uint32_t num_threads); + int get_num_hardware_threads(); + } // namespace basisu #include "basisu_math.h" diff --git a/encoder/basisu_etc.cpp b/encoder/basisu_etc.cpp index abc78e3..5bae228 100644 --- a/encoder/basisu_etc.cpp +++ b/encoder/basisu_etc.cpp @@ -1,5 +1,5 @@ // basis_etc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ namespace basisu { -16,-48,-64,-80,8,40,56,72 }, { -16,-40,-64,-80,8,32,56,72 }, { -16,-32,-64,-80,8,24,56,72 }, { -16,-40,-56,-80,8,32,48,72 }, { -24,-32,-56,-80,16,24,48,72 }, { -8,-16,-24,-80,0,8,16,72 }, { -32,-48,-64,-72,24,40,56,64 }, { -24,-40,-56,-72,16,32,48,64 } }; - + // Given an ETC1 diff/inten_table/selector, and an 8-bit desired color, this table encodes the best packed_color in the low byte, and the abs error in the high byte. static uint16_t g_etc1_inverse_lookup[2 * 8 * 4][256]; // [ diff/inten_table/selector][desired_color ] @@ -113,7 +113,7 @@ namespace basisu static uint32_t etc1_decode_value(uint32_t diff, uint32_t inten, uint32_t selector, uint32_t packed_c) { - const uint32_t limit = diff ? 32 : 16; + const uint32_t limit = diff ? 32 : 16; BASISU_NOTE_UNUSED(limit); assert((diff < 2) && (inten < 8) && (selector < 4) && (packed_c < limit)); int c; @@ -261,7 +261,7 @@ namespace basisu return best_error; } - + const uint32_t BASISU_ETC1_CLUSTER_FIT_ORDER_TABLE_SIZE = 165; static const struct { uint8_t m_v[4]; } g_cluster_fit_order_tab[BASISU_ETC1_CLUSTER_FIT_ORDER_TABLE_SIZE] = @@ -300,7 +300,7 @@ namespace basisu { { 2, 1, 2, 3 } },{ { 4, 1, 0, 3 } },{ { 3, 1, 1, 3 } },{ { 1, 1, 2, 4 } },{ { 2, 1, 0, 5 } }, { { 1, 0, 1, 6 } },{ { 0, 2, 1, 5 } },{ { 0, 2, 0, 6 } },{ { 1, 1, 1, 5 } },{ { 1, 1, 0, 6 } } }; - + const int g_etc1_inten_tables[cETC1IntenModifierValues][cETC1SelectorValues] = { { -8, -2, 2, 8 }, { -17, -5, 5, 17 }, { -29, -9, 9, 29 }, { -42, -13, 13, 42 }, @@ -600,7 +600,7 @@ namespace basisu const int y3 = pInten_modifer_table[3]; pDst[3].set(ir + y3, ig + y3, ib + y3, 255); } - + bool unpack_etc1(const etc_block& block, color_rgba *pDst, bool preserve_alpha) { const bool diff_flag = block.get_diff_bit(); @@ -723,7 +723,7 @@ namespace basisu { return (n << 4) | n; } - + uint64_t etc_block::evaluate_etc1_error(const color_rgba* pBlock_pixels, bool perceptual, int subblock_index) const { color_rgba unpacked_block[16]; @@ -772,7 +772,7 @@ namespace basisu } } } - + bool etc1_optimizer::compute() { assert(m_pResult->m_pSelectors); @@ -811,26 +811,26 @@ namespace basisu #if defined(DEBUG) || defined(_DEBUG) { - // Ultimate sanity check on the returned error. + // Ultimate sanity check on the returned error. // If this check fails, it likely means the SSE code diverged from C++ somehow, or there was an overflow somewhere. color_rgba block_colors[4]; m_best_solution.m_coords.get_block_colors(block_colors); const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; uint64_t actual_error = 0; - + bool perceptual; if (m_pParams->m_quality >= cETCQualityMedium) perceptual = m_pParams->m_perceptual; else perceptual = (m_pParams->m_quality == cETCQualityFast) ? false : m_pParams->m_perceptual; - + for (uint32_t i = 0; i < n; i++) actual_error += color_distance(perceptual, pSrc_pixels[i], block_colors[pSelectors[i]], false); - + assert(actual_error == m_best_solution.m_error); } -#endif +#endif m_pResult->m_error = m_best_solution.m_error; @@ -1015,10 +1015,10 @@ namespace basisu m_luma.resize(n); m_sorted_luma_indices.resize(n); m_sorted_luma.resize(n); - + int min_r = 255, min_g = 255, min_b = 255; int max_r = 0, max_g = 0, max_b = 0; - + for (uint32_t i = 0; i < n; i++) { const color_rgba& c = m_pParams->m_pSrc_pixels[i]; @@ -1056,7 +1056,7 @@ namespace basisu m_pSorted_luma = &m_sorted_luma[0]; m_pSorted_luma_indices = &m_sorted_luma_indices[0]; - + for (uint32_t i = 0; i < n; i++) m_pSorted_luma[i] = m_luma[m_pSorted_luma_indices[i]]; } @@ -1072,7 +1072,7 @@ namespace basisu bool etc1_optimizer::check_for_redundant_solution(const etc1_solution_coordinates& coords) { // Hash first 3 bytes of color (RGB) - uint32_t kh = hash_hsieh((uint8_t*)&coords.m_unscaled_color.r, 3); + uint32_t kh = basist::hash_hsieh((uint8_t*)&coords.m_unscaled_color.r, 3); uint32_t h0 = kh & cSolutionsTriedHashMask; uint32_t h1 = (kh >> cSolutionsTriedHashBits) & cSolutionsTriedHashMask; @@ -1087,7 +1087,7 @@ namespace basisu return true; } - + static uint8_t g_eval_dist_tables[8][256] = { // 99% threshold @@ -1178,7 +1178,7 @@ namespace basisu uint64_t total_error = 0; const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; - + if (!g_cpu_supports_sse41) { for (uint32_t c = 0; c < n; c++) @@ -1235,6 +1235,7 @@ namespace basisu perceptual_distance_rgb_4_N_sse41((int64_t*)&total_error, pSelectors_to_use, block_colors, pSrc_pixels, n, trial_solution.m_error); else linear_distance_rgb_4_N_sse41((int64_t*)&total_error, pSelectors_to_use, block_colors, pSrc_pixels, n, trial_solution.m_error); + for (uint32_t i = 0; i < n; i++) m_temp_selectors[i] = pSelectors_to_use[i]; } @@ -1258,7 +1259,7 @@ namespace basisu } trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color; trial_solution.m_coords.m_color4 = m_pParams->m_use_color4; - + #if BASISU_DEBUG_ETC_ENCODER_DEEPER printf("Eval done: %u error: %I64u best error so far: %I64u\n", (trial_solution.m_error < pBest_solution->m_error), trial_solution.m_error, pBest_solution->m_error); #endif @@ -1272,7 +1273,7 @@ namespace basisu success = true; } } - + return success; } @@ -1303,14 +1304,14 @@ namespace basisu } const color_rgba base_color(coords.get_scaled_color()); - + const uint32_t n = m_pParams->m_num_src_pixels; assert(trial_solution.m_selectors.size() == n); trial_solution.m_error = UINT64_MAX; - + const bool perceptual = (m_pParams->m_quality == cETCQualityFast) ? false : m_pParams->m_perceptual; - + for (int inten_table = cETC1IntenModifierValues - 1; inten_table >= 0; --inten_table) { const int* pInten_table = g_etc1_inten_tables[inten_table]; @@ -1330,10 +1331,10 @@ namespace basisu // 0 1 2 3 // 01 12 23 const uint32_t block_inten_midpoints[3] = { block_inten[0] + block_inten[1], block_inten[1] + block_inten[2], block_inten[2] + block_inten[3] }; - + uint64_t total_error = 0; const color_rgba* pSrc_pixels = m_pParams->m_pSrc_pixels; - + if (perceptual) { if ((m_pSorted_luma[n - 1] * 2) < block_inten_midpoints[0]) diff --git a/encoder/basisu_frontend.cpp b/encoder/basisu_frontend.cpp index 102f1bd..d721b37 100644 --- a/encoder/basisu_frontend.cpp +++ b/encoder/basisu_frontend.cpp @@ -1,5 +1,5 @@ // basisu_frontend.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// TODO: +// TODO: // This code originally supported full ETC1 and ETC1S, so there's some legacy stuff to be cleaned up in here. // Add endpoint tiling support (where we force adjacent blocks to use the same endpoints during quantization), for a ~10% or more increase in bitrate at same SSIM. The backend already supports this. // @@ -40,20 +40,20 @@ namespace basisu const uint32_t BASISU_ENDPOINT_PARENT_CODEBOOK_SIZE = 16; const uint32_t BASISU_SELECTOR_PARENT_CODEBOOK_SIZE_COMP_LEVEL_01 = 32; const uint32_t BASISU_SELECTOR_PARENT_CODEBOOK_SIZE_COMP_LEVEL_DEFAULT = 16; - + // TODO - How to handle internal verifies in the basisu lib static inline void handle_verify_failure(int line) { error_printf("basisu_frontend: verify check failed at line %i!\n", line); abort(); } - + bool basisu_frontend::init(const params &p) { debug_printf("basisu_frontend::init: Multithreaded: %u, Job pool total threads: %u, NumEndpointClusters: %u, NumSelectorClusters: %u, Perceptual: %u, CompressionLevel: %u\n", p.m_multithreaded, p.m_pJob_pool ? p.m_pJob_pool->get_total_threads() : 0, p.m_max_endpoint_clusters, p.m_max_selector_clusters, p.m_perceptual, p.m_compression_level); - + if ((p.m_max_endpoint_clusters < 1) || (p.m_max_endpoint_clusters > cMaxEndpointClusters)) return false; if ((p.m_max_selector_clusters < 1) || (p.m_max_selector_clusters > cMaxSelectorClusters)) @@ -61,9 +61,9 @@ namespace basisu m_source_blocks.resize(0); append_vector(m_source_blocks, p.m_pSource_blocks, p.m_num_source_blocks); - + m_params = p; - + if (m_params.m_pOpenCL_context) { BASISU_ASSUME(sizeof(cl_pixel_block) == sizeof(pixel_block)); @@ -80,7 +80,7 @@ namespace basisu m_encoded_blocks.resize(m_params.m_num_source_blocks); memset(&m_encoded_blocks[0], 0, m_encoded_blocks.size() * sizeof(m_encoded_blocks[0])); - + m_num_endpoint_codebook_iterations = 1; m_num_selector_codebook_iterations = 1; @@ -150,7 +150,7 @@ namespace basisu if (m_params.m_disable_hierarchical_endpoint_codebooks) m_use_hierarchical_endpoint_codebooks = false; - debug_printf("Endpoint refinement: %u, Hierarchical endpoint codebooks: %u, Hierarchical selector codebooks: %u, Endpoint codebook iters: %u, Selector codebook iters: %u\n", + debug_printf("Endpoint refinement: %u, Hierarchical endpoint codebooks: %u, Hierarchical selector codebooks: %u, Endpoint codebook iters: %u, Selector codebook iters: %u\n", m_endpoint_refinement, m_use_hierarchical_endpoint_codebooks, m_use_hierarchical_selector_codebooks, m_num_endpoint_codebook_iterations, m_num_selector_codebook_iterations); return true; @@ -238,7 +238,7 @@ namespace basisu { BASISU_FRONTEND_VERIFY(validate_endpoint_cluster_hierarchy(false)); } - + eliminate_redundant_or_empty_endpoint_clusters(); if (m_params.m_validate) @@ -252,7 +252,7 @@ namespace basisu if (early_out) break; } - + if (m_params.m_validate) { BASISU_FRONTEND_VERIFY(check_etc1s_constraints()); @@ -268,13 +268,13 @@ namespace basisu if (m_use_hierarchical_selector_codebooks) compute_selector_clusters_within_each_parent_cluster(); - + if (m_params.m_compression_level == 0) { create_optimized_selector_codebook(0); find_optimal_selector_clusters_for_each_block(); - + introduce_special_selector_clusters(); } else @@ -295,7 +295,7 @@ namespace basisu } } } - + optimize_selector_codebook(); if (m_params.m_debug_stats) @@ -321,7 +321,7 @@ namespace basisu const basist::basisu_lowlevel_etc1s_transcoder::endpoint_vec& endpoints = pTranscoder->get_endpoints(); const basist::basisu_lowlevel_etc1s_transcoder::selector_vec& selectors = pTranscoder->get_selectors(); - + m_endpoint_cluster_etc_params.resize(endpoints.size()); for (uint32_t i = 0; i < endpoints.size(); i++) { @@ -420,7 +420,7 @@ namespace basisu const uint32_t last_index = minimum(m_total_blocks, first_index + N); m_params.m_pJob_pool->add_job([this, first_index, last_index, pass] { - + for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const etc_block& blk = pass ? m_encoded_blocks[block_index] : m_etc1_blocks_etc1s[block_index]; @@ -433,7 +433,7 @@ namespace basisu uint64_t best_err = UINT64_MAX; uint32_t best_index = 0; etc_block best_block(trial_blk); - + for (uint32_t i = 0; i < m_endpoint_cluster_etc_params.size(); i++) { if (m_endpoint_cluster_etc_params[i].m_inten_table[0] > blk.get_inten_table(0)) @@ -496,6 +496,7 @@ namespace basisu const uint32_t block_endpoint_index = m_block_endpoint_clusters_indices[block_index][0]; etc_block trial_blk; + clear_obj(trial_blk); trial_blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_color_unscaled[0]); trial_blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_inten_table[0]); trial_blk.set_flip_bit(true); @@ -546,7 +547,7 @@ namespace basisu m_selector_cluster_block_indices.resize(selectors.size()); for (uint32_t block_index = 0; block_index < m_etc1_blocks_etc1s.size(); block_index++) m_selector_cluster_block_indices[m_block_selector_cluster_index[block_index]].push_back(block_index); - + return true; } @@ -580,9 +581,9 @@ namespace basisu const uint32_t new_selector_cluster_index = m_optimized_cluster_selectors.size_u32(); m_optimized_cluster_selectors.push_back(blk); - + vector_ensure_element_is_valid(m_selector_cluster_block_indices, new_selector_cluster_index); - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { if (m_orig_encoded_blocks[block_index].get_raw_selector_bits() != blk.get_raw_selector_bits()) @@ -590,7 +591,7 @@ namespace basisu // See if using flat selectors actually decreases the block's error. const uint32_t old_selector_cluster_index = m_block_selector_cluster_index[block_index]; - + etc_block cur_blk; const uint32_t endpoint_cluster_index = get_subblock_endpoint_cluster_index(block_index, 0); cur_blk.set_block_color5_etc1s(get_endpoint_cluster_unscaled_color(endpoint_cluster_index, false)); @@ -606,10 +607,10 @@ namespace basisu if (new_err >= cur_err) continue; - + // Change the block to use the new cluster m_block_selector_cluster_index[block_index] = new_selector_cluster_index; - + m_selector_cluster_block_indices[new_selector_cluster_index].push_back(block_index); block_relocated_flags[block_index] = true; @@ -685,7 +686,7 @@ namespace basisu old_to_new[i] = (find_res.first)->second; continue; } - + old_to_new[i] = total_new_entries++; new_to_old.push_back(i); } @@ -714,7 +715,7 @@ namespace basisu { new_selector_cluster_indices[m_block_selector_cluster_index[i]].push_back(i); } - + m_optimized_cluster_selectors.swap(new_optimized_cluster_selectors); m_selector_cluster_block_indices.swap(new_selector_cluster_indices); @@ -725,7 +726,7 @@ namespace basisu for (uint32_t j = 0; j < m_selector_clusters_within_each_parent_cluster[i].size(); j++) m_selector_clusters_within_each_parent_cluster[i][j] = old_to_new[m_selector_clusters_within_each_parent_cluster[i][j]]; } - + debug_printf("optimize_selector_codebook: Before: %u After: %u\n", orig_total_selector_clusters, total_new_entries); } @@ -735,11 +736,11 @@ namespace basisu interval_timer tm; tm.start(); - + m_etc1_blocks_etc1s.resize(m_total_blocks); bool use_cpu = true; - + if (m_params.m_pOpenCL_context) { uint32_t total_perms = 64; @@ -747,9 +748,9 @@ namespace basisu total_perms = 4; else if (m_params.m_compression_level == 1) total_perms = 16; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) total_perms = OPENCL_ENCODE_ETC1S_MAX_PERMS; - + bool status = opencl_encode_etc1s_blocks(m_params.m_pOpenCL_context, m_etc1_blocks_etc1s.data(), m_params.m_perceptual, total_perms); if (status) use_cpu = false; @@ -760,7 +761,7 @@ namespace basisu m_opencl_failed = true; } } - + if (use_cpu) { const uint32_t N = 4096; @@ -783,7 +784,7 @@ namespace basisu optimizer_params.m_quality = cETCQualityFast; else if (m_params.m_compression_level == 1) optimizer_params.m_quality = cETCQualityMedium; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) optimizer_params.m_quality = cETCQualityUber; optimizer_params.m_num_src_pixels = 16; @@ -817,16 +818,16 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); } // use_cpu - + debug_printf("init_etc1_images: Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); } void basisu_frontend::init_endpoint_training_vectors() { debug_printf("init_endpoint_training_vectors\n"); - + vec6F_quantizer::array_of_weighted_training_vecs &training_vecs = m_endpoint_clusterizer.get_training_vecs(); - + training_vecs.resize(m_total_blocks * 2); const uint32_t N = 16384; @@ -838,12 +839,12 @@ namespace basisu m_params.m_pJob_pool->add_job( [this, first_index, last_index, &training_vecs] { for (uint32_t block_index = first_index; block_index < last_index; block_index++) - { + { const etc_block &blk = m_etc1_blocks_etc1s[block_index]; color_rgba block_colors[2]; blk.get_block_low_high_colors(block_colors, 0); - + vec6F v; v[0] = block_colors[0].r * (1.0f / 255.0f); v[1] = block_colors[0].g * (1.0f / 255.0f); @@ -851,7 +852,7 @@ namespace basisu v[3] = block_colors[1].r * (1.0f / 255.0f); v[4] = block_colors[1].g * (1.0f / 255.0f); v[5] = block_colors[1].b * (1.0f / 255.0f); - + training_vecs[block_index * 2 + 0] = std::make_pair(v, 1); training_vecs[block_index * 2 + 1] = std::make_pair(v, 1); @@ -870,7 +871,7 @@ namespace basisu const uint32_t parent_codebook_size = (m_params.m_max_endpoint_clusters >= 256) ? BASISU_ENDPOINT_PARENT_CODEBOOK_SIZE : 0; uint32_t max_threads = 0; - max_threads = m_params.m_multithreaded ? minimum(std::thread::hardware_concurrency(), cMaxCodebookCreationThreads) : 0; + max_threads = m_params.m_multithreaded ? minimum(get_num_hardware_threads(), cMaxCodebookCreationThreads) : 0; if (m_params.m_pJob_pool) max_threads = minimum((int)m_params.m_pJob_pool->get_total_threads(), max_threads); @@ -919,12 +920,12 @@ namespace basisu for (uint32_t cluster_index = 0; cluster_index < m_endpoint_clusters.size(); cluster_index++) { const uint_vec &cluster = m_endpoint_clusters[cluster_index]; - + uint32_t parent_cluster_index = 0; for (uint32_t j = 0; j < cluster.size(); j++) { const uint32_t block_index = cluster[j] >> 1; - + BASISU_FRONTEND_VERIFY(block_index < m_block_parent_endpoint_cluster.size()); if (!j) @@ -938,7 +939,7 @@ namespace basisu } } } - + if (m_params.m_debug_stats) debug_printf("Total endpoint clusters: %u, parent clusters: %u\n", m_endpoint_clusters.size_u32(), m_endpoint_parent_clusters.size_u32()); } @@ -996,7 +997,7 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_indices.size()); vector_sort(cluster_indices); - + auto last = std::unique(cluster_indices.begin(), cluster_indices.end()); cluster_indices.erase(last, cluster_indices.end()); } @@ -1009,8 +1010,8 @@ namespace basisu const uint32_t N = 512; for (uint32_t cluster_index_iter = 0; cluster_index_iter < m_endpoint_clusters.size(); cluster_index_iter += N) { - const uint32_t first_index = cluster_index_iter; - const uint32_t last_index = minimum(m_endpoint_clusters.size_u32(), cluster_index_iter + N); + const uint32_t first_index = cluster_index_iter; + const uint32_t last_index = minimum(m_endpoint_clusters.size_u32(), cluster_index_iter + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index] { @@ -1039,7 +1040,7 @@ namespace basisu const endpoint_cluster_etc_params &etc_params = m_endpoint_cluster_etc_params[cluster_index]; assert(etc_params.m_valid); - + color_rgba block_colors[4]; etc_block::get_block_colors5(block_colors, etc_params.m_color_unscaled[0], etc_params.m_inten_table[0], true); @@ -1071,7 +1072,7 @@ namespace basisu quant_err.m_cluster_subblock_index = cluster_indices_iter; quant_err.m_block_index = block_index; quant_err.m_subblock_index = subblock_index; - + { std::lock_guard lock(m_lock); @@ -1088,7 +1089,7 @@ namespace basisu vector_sort(m_subblock_endpoint_quant_err_vec); } - + void basisu_frontend::introduce_new_endpoint_clusters() { debug_printf("introduce_new_endpoint_clusters\n"); @@ -1159,9 +1160,9 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_sizes[subblock_to_move.m_cluster_index] >= 2); cluster_sizes[subblock_to_move.m_cluster_index] -= 2; - + ignore_cluster.insert(subblock_to_move.m_cluster_index); - + total_new_clusters++; num_new_endpoint_clusters--; @@ -1197,23 +1198,23 @@ namespace basisu inline std::size_t operator()(const color_rgba& k) const { uint32_t v = *(const uint32_t*)&k; - + //return bitmix32(v); - + //v ^= (v << 10); //v ^= (v >> 12); - + return v; } }; - + // Given each endpoint cluster, gather all the block pixels which are in that cluster and compute optimized ETC1S endpoints for them. // TODO: Don't optimize endpoint clusters which haven't changed. // If step>=1, we check to ensure the new endpoint values actually decrease quantization error. void basisu_frontend::generate_endpoint_codebook(uint32_t step) { debug_printf("generate_endpoint_codebook\n"); - + interval_timer tm; tm.start(); @@ -1226,7 +1227,7 @@ namespace basisu const uint32_t total_clusters = (uint32_t)m_endpoint_clusters.size(); basisu::vector pixel_clusters(total_clusters); - + std::vector input_pixels; input_pixels.reserve(m_total_blocks * 16); @@ -1259,7 +1260,7 @@ namespace basisu pixel_weights.resize(pixel_weights.size() + total_pixels); uint64_t dst_ofs = first_pixel_index; - + uint64_t total_r = 0, total_g = 0, total_b = 0; for (uint32_t cluster_indices_iter = 0; cluster_indices_iter < cluster_indices.size(); cluster_indices_iter++) { @@ -1311,7 +1312,7 @@ namespace basisu const uint64_t first_pixel_index = input_pixels.size(); uint32_t prev_color = 0, cur_weight = 0; - + for (uint32_t i = 0; i < colors.size(); i++) { uint32_t cur_color = pSorted[i]; @@ -1359,7 +1360,7 @@ namespace basisu uint32_t *pPrev_weight = nullptr; color_rgba prev_color; - + { color_rgba cur_color = pBlock_pixels[0]; auto res = color_hasher.insert(cur_color, 0); @@ -1371,7 +1372,7 @@ namespace basisu prev_color = cur_color; pPrev_weight = &(res.first)->second; } - + for (uint32_t i = 1; i < 16; i++) { color_rgba cur_color = pBlock_pixels[i]; @@ -1404,9 +1405,9 @@ namespace basisu input_pixels.resize(first_pixel_index + total_unique_pixels); pixel_weights.resize(first_pixel_index + total_unique_pixels); - + uint32_t j = 0; - + for (auto it = color_hasher.begin(); it != color_hasher.end(); ++it, ++j) { input_pixels[first_pixel_index + j] = it->first; @@ -1438,7 +1439,7 @@ namespace basisu uint32_t total_perms = 64; if (m_params.m_compression_level <= 1) total_perms = 16; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) total_perms = OPENCL_ENCODE_ETC1S_MAX_PERMS; basisu::vector output_blocks(total_clusters); @@ -1456,7 +1457,7 @@ namespace basisu for (uint32_t old_cluster_index = 0; old_cluster_index < m_endpoint_clusters.size(); old_cluster_index++) { const uint32_t new_cluster_index = sorted_cluster_indices_old_to_new[old_cluster_index]; - + const etc_block& blk = output_blocks[new_cluster_index]; endpoint_cluster_etc_params& prev_etc_params = m_endpoint_cluster_etc_params[old_cluster_index]; @@ -1464,7 +1465,7 @@ namespace basisu prev_etc_params.m_valid = true; etc_block::unpack_color5(prev_etc_params.m_color_unscaled[0], blk.get_base5_color(), false); prev_etc_params.m_inten_table[0] = blk.get_inten_table(0); - prev_etc_params.m_color_error[0] = 0; // dummy value - we don't actually use this + prev_etc_params.m_color_error[0] = 0; // dummy value - we don't actually use this } use_cpu = false; @@ -1518,7 +1519,7 @@ namespace basisu { etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -1529,7 +1530,7 @@ namespace basisu if (m_params.m_compression_level <= 1) cluster_optimizer_params.m_quality = cETCQualityMedium; - else if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + else if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) cluster_optimizer_params.m_quality = cETCQualityUber; etc1_optimizer::results cluster_optimizer_results; @@ -1647,7 +1648,7 @@ namespace basisu uint32_t basisu_frontend::refine_endpoint_clusterization() { debug_printf("refine_endpoint_clusterization\n"); - + if (m_use_hierarchical_endpoint_codebooks) compute_endpoint_clusters_within_each_parent_cluster(); @@ -1668,9 +1669,9 @@ namespace basisu } // cluster_indices_iter } - + //---------------------------------------------------------- - + // Create a new endpoint clusterization interval_timer tm; @@ -1687,7 +1688,7 @@ namespace basisu const uint32_t total_parent_clusters = (uint32_t)m_endpoint_clusters_within_each_parent_cluster.size(); basisu::vector cl_block_info_structs(m_total_blocks); - + // the size of each parent cluster, in total clusters uint_vec parent_cluster_sizes(total_parent_clusters); for (uint32_t i = 0; i < total_parent_clusters; i++) @@ -1701,7 +1702,7 @@ namespace basisu cur_ofs += parent_cluster_sizes[i]; } - + // Note: total_actual_endpoint_clusters is not necessarly equal to m_endpoint_clusters.size(), because clusters may live in multiple parent clusters after the first refinement step. BASISU_FRONTEND_VERIFY(cur_ofs >= m_endpoint_clusters.size()); const uint32_t total_actual_endpoint_clusters = cur_ofs; @@ -1727,11 +1728,11 @@ namespace basisu cl_endpoint_cluster_structs[dst_ofs + j].m_cluster_index = (uint16_t)endpoint_cluster_index; } } - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { const uint32_t block_parent_endpoint_cluster_index = m_block_parent_endpoint_cluster[block_index]; - + cl_block_info_structs[block_index].m_num_clusters = (uint16_t)(parent_cluster_sizes[block_parent_endpoint_cluster_index]); cl_block_info_structs[block_index].m_first_cluster_ofs = (uint16_t)(first_parent_cluster_ofs[block_parent_endpoint_cluster_index]); @@ -1746,7 +1747,7 @@ namespace basisu uint_vec sorted_block_indices(m_total_blocks); indirect_sort(m_total_blocks, sorted_block_indices.data(), block_cluster_indices.data()); - + bool status = opencl_refine_endpoint_clusterization( m_params.m_pOpenCL_context, cl_block_info_structs.data(), @@ -1902,7 +1903,7 @@ namespace basisu break; } } // j - + best_cluster_indices[block_index] = best_cluster_index; } // block_index @@ -1912,9 +1913,9 @@ namespace basisu } // block_index_iter m_params.m_pJob_pool->wait_for_all(); - + } // use_cpu - + debug_printf("refine_endpoint_clusterization time: %3.3f secs\n", tm.get_elapsed_secs()); basisu::vector > optimized_endpoint_clusters(m_endpoint_clusters.size()); @@ -1957,7 +1958,7 @@ namespace basisu basisu::vector > new_endpoint_clusters(m_endpoint_clusters.size()); basisu::vector new_subblock_etc_params(m_endpoint_clusters.size()); - + for (uint32_t i = 0; i < m_endpoint_clusters.size(); i++) { uint32_t j = sorted_endpoint_cluster_indices[i]; @@ -1972,7 +1973,7 @@ namespace basisu new_endpoint_clusters.resize(0); new_subblock_etc_params.resize(0); - + for (int i = 0; i < (int)m_endpoint_clusters.size(); ) { if (!m_endpoint_clusters[i].size()) @@ -1990,7 +1991,7 @@ namespace basisu new_endpoint_clusters.push_back(m_endpoint_clusters[i]); new_subblock_etc_params.push_back(m_endpoint_cluster_etc_params[i]); - + for (int k = i + 1; k < j; k++) { append_vector(new_endpoint_clusters.back(), m_endpoint_clusters[k]); @@ -1998,7 +1999,7 @@ namespace basisu i = j; } - + if (m_endpoint_clusters.size() != new_endpoint_clusters.size()) { if (m_params.m_debug_stats) @@ -2013,7 +2014,7 @@ namespace basisu void basisu_frontend::create_initial_packed_texture() { debug_printf("create_initial_packed_texture\n"); - + interval_timer tm; tm.start(); @@ -2026,7 +2027,7 @@ namespace basisu for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { uint32_t cluster0 = m_block_endpoint_clusters_indices[block_index][0]; - + const color_rgba& color_unscaled = m_endpoint_cluster_etc_params[cluster0].m_color_unscaled[0]; uint32_t inten = m_endpoint_cluster_etc_params[cluster0].m_inten_table[0]; @@ -2088,7 +2089,7 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); } // use_cpu - + m_orig_encoded_blocks = m_encoded_blocks; debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); @@ -2105,7 +2106,7 @@ namespace basisu for (uint32_t cluster_indices_iter = 0; cluster_indices_iter < cluster_indices.size(); cluster_indices_iter++) { const uint32_t block_index = cluster_indices[cluster_indices_iter]; - + block_selector_cluster_indices[block_index] = cluster_index; } // cluster_indices_iter @@ -2130,7 +2131,7 @@ namespace basisu BASISU_FRONTEND_VERIFY(cluster_indices.size()); vector_sort(cluster_indices); - + auto last = std::unique(cluster_indices.begin(), cluster_indices.end()); cluster_indices.erase(last, cluster_indices.end()); } @@ -2139,11 +2140,11 @@ namespace basisu void basisu_frontend::generate_selector_clusters() { debug_printf("generate_selector_clusters\n"); - + typedef tree_vector_quant vec16F_clusterizer; - + vec16F_clusterizer::array_of_weighted_training_vecs training_vecs(m_total_blocks); - + const uint32_t N = 4096; for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N) { @@ -2171,10 +2172,10 @@ namespace basisu const uint32_t cColorDistToWeight = 300; const uint32_t cMaxWeight = 4096; uint32_t weight = clamp(dist / cColorDistToWeight, 1, cMaxWeight); - + training_vecs[block_index].first = v; training_vecs[block_index].second = weight; - + } // block_index } ); @@ -2192,7 +2193,7 @@ namespace basisu debug_printf("Using selector parent codebook size %u\n", parent_codebook_size); uint32_t max_threads = 0; - max_threads = m_params.m_multithreaded ? minimum(std::thread::hardware_concurrency(), cMaxCodebookCreationThreads) : 0; + max_threads = m_params.m_multithreaded ? minimum(get_num_hardware_threads(), cMaxCodebookCreationThreads) : 0; if (m_params.m_pJob_pool) max_threads = minimum((int)m_params.m_pJob_pool->get_total_threads(), max_threads); @@ -2235,7 +2236,7 @@ namespace basisu for (uint32_t cluster_index = 0; cluster_index < m_selector_cluster_block_indices.size(); cluster_index++) { const uint_vec &cluster = m_selector_cluster_block_indices[cluster_index]; - + uint32_t parent_cluster_index = 0; for (uint32_t j = 0; j < cluster.size(); j++) { @@ -2267,7 +2268,7 @@ namespace basisu debug_printf("Total selector clusters (from m_selector_cluster_block_indices.size()): %u\n", (uint32_t)m_selector_cluster_block_indices.size()); m_optimized_cluster_selectors.resize(total_selector_clusters); - + // For each selector codebook entry, and for each of the 4x4 selectors, determine which selector minimizes the error across all the blocks that use that quantized selector. const uint32_t N = 256; for (uint32_t cluster_index_iter = 0; cluster_index_iter < total_selector_clusters; cluster_index_iter += N) @@ -2351,7 +2352,7 @@ namespace basisu m_params.m_pJob_pool->wait_for_all(); debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); - + if (m_params.m_debug_images) { uint32_t max_selector_cluster_size = 0; @@ -2377,7 +2378,7 @@ namespace basisu uint32_t block_index = cluster_block_indices[i]; const etc_block &blk = m_orig_encoded_blocks[block_index]; - + for (uint32_t y = 0; y < 4; y++) for (uint32_t x = 0; x < 4; x++) selector_cluster_vis.set_clipped(x_spacer_len + x + 5 * i, selector_cluster_index * 5 + y, color_rgba((blk.get_selector(x, y) * 255) / 3)); @@ -2399,7 +2400,7 @@ namespace basisu interval_timer tm; tm.start(); - + if (m_params.m_validate) { // Sanity checks @@ -2414,7 +2415,7 @@ namespace basisu } m_block_selector_cluster_index.resize(m_total_blocks); - + if (m_params.m_compression_level == 0) { // Just leave the blocks in their original selector clusters. @@ -2435,7 +2436,7 @@ namespace basisu return; } - + bool use_cpu = true; if ((m_params.m_pOpenCL_context) && m_use_hierarchical_selector_codebooks) @@ -2444,17 +2445,17 @@ namespace basisu basisu::vector selector_structs; selector_structs.reserve(m_optimized_cluster_selectors.size()); - + uint_vec parent_selector_cluster_offsets(num_parent_clusters); uint_vec selector_cluster_indices; selector_cluster_indices.reserve(m_optimized_cluster_selectors.size()); - + uint32_t cur_ofs = 0; for (uint32_t parent_index = 0; parent_index < num_parent_clusters; parent_index++) { parent_selector_cluster_offsets[parent_index] = cur_ofs; - + for (uint32_t j = 0; j < m_selector_clusters_within_each_parent_cluster[parent_index].size(); j++) { const uint32_t selector_cluster_index = m_selector_clusters_within_each_parent_cluster[parent_index][j]; @@ -2464,7 +2465,7 @@ namespace basisu sel_bits |= (m_optimized_cluster_selectors[selector_cluster_index].get_selector(p & 3, p >> 2) << (p * 2)); selector_structs.enlarge(1)->m_packed_selectors = sel_bits; - + selector_cluster_indices.push_back(selector_cluster_index); } @@ -2472,7 +2473,7 @@ namespace basisu } const uint32_t total_input_selectors = cur_ofs; - + basisu::vector block_structs(m_total_blocks); for (uint32_t i = 0; i < m_total_blocks; i++) { @@ -2496,7 +2497,7 @@ namespace basisu selector_cluster_indices.data(), output_selector_cluster_indices.data(), m_params.m_perceptual); - + if (!status) { error_printf("basisu_frontend::find_optimal_selector_clusters_for_each_block: opencl_find_optimal_selector_clusters_for_each_block() failed! Using CPU.\n"); @@ -2510,7 +2511,7 @@ namespace basisu m_selector_cluster_block_indices[i].resize(0); m_selector_cluster_block_indices[i].reserve(128); } - + for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++) { etc_block& blk = m_encoded_blocks[block_index]; @@ -2542,7 +2543,7 @@ namespace basisu } } } - + const uint32_t N = 2048; for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N) { @@ -2550,13 +2551,13 @@ namespace basisu const uint32_t last_index = minimum(m_total_blocks, first_index + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index, &unpacked_optimized_cluster_selectors] { - + int prev_best_cluster_index = 0; for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const pixel_block& block = get_source_pixel_block(block_index); - + etc_block& blk = m_encoded_blocks[block_index]; if ((block_index > first_index) && (block == get_source_pixel_block(block_index - 1))) @@ -2564,18 +2565,18 @@ namespace basisu blk.set_raw_selector_bits(m_optimized_cluster_selectors[prev_best_cluster_index].get_raw_selector_bits()); m_block_selector_cluster_index[block_index] = prev_best_cluster_index; - + continue; } - + const color_rgba* pBlock_pixels = block.get_ptr(); - + color_rgba trial_block_colors[4]; blk.get_block_colors_etc1s(trial_block_colors); // precompute errors for the i-th block pixel and selector sel: [sel][i] uint32_t trial_errors[4][16]; - + if (m_params.m_perceptual) { for (uint32_t sel = 0; sel < 4; ++sel) @@ -2652,7 +2653,7 @@ namespace basisu for (uint32_t cluster_iter = 0; cluster_iter < total_clusters; cluster_iter++) { const uint32_t cluster_index = m_use_hierarchical_selector_codebooks ? (*pCluster_indices)[cluster_iter] : cluster_iter; - + const uint8_t* pSels = &unpacked_optimized_cluster_selectors[cluster_index * 16]; uint64_t trial_err = (uint64_t)trial_errors[pSels[0]][0] + trial_errors[pSels[1]][1] + trial_errors[pSels[2]][2] + trial_errors[pSels[3]][3]; @@ -2685,7 +2686,7 @@ namespace basisu m_block_selector_cluster_index[block_index] = best_cluster_index; prev_best_cluster_index = best_cluster_index; - + } // block_index } ); @@ -2693,7 +2694,7 @@ namespace basisu } // block_index_iter m_params.m_pJob_pool->wait_for_all(); - + for (uint32_t i = 0; i < m_selector_cluster_block_indices.size(); i++) { m_selector_cluster_block_indices[i].resize(0); @@ -2707,7 +2708,7 @@ namespace basisu vector_ensure_element_is_valid(m_selector_cluster_block_indices, best_cluster_index); m_selector_cluster_block_indices[best_cluster_index].push_back(block_index); } - + } // if (use_cpu) debug_printf("Elapsed time: %3.3f secs\n", tm.get_elapsed_secs()); @@ -2717,7 +2718,7 @@ namespace basisu uint32_t basisu_frontend::refine_block_endpoints_given_selectors() { debug_printf("refine_block_endpoints_given_selectors\n"); - + for (int block_index = 0; block_index < static_cast(m_total_blocks); block_index++) { //uint32_t selector_cluster = m_block_selector_cluster_index(block_x, block_y); @@ -2790,7 +2791,7 @@ namespace basisu total_subblocks_examined += total_pixels / 8; etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -2898,7 +2899,7 @@ namespace basisu if (m_params.m_debug_stats) debug_printf("Total subblock endpoints refined: %u (%3.1f%%)\n", total_subblocks_refined, total_subblocks_refined * 100.0f / total_subblocks_examined); - + return total_subblocks_refined; } @@ -2990,7 +2991,7 @@ namespace basisu } // The backend has remapped the block endpoints while optimizing the output symbols for better rate distortion performance, so let's go and reoptimize the endpoint codebook. - // This is currently the only place where the backend actually goes and changes the quantization and calls the frontend to fix things up. + // This is currently the only place where the backend actually goes and changes the quantization and calls the frontend to fix things up. // This is basically a bottom up clusterization stage, where some leaves can be combined. void basisu_frontend::reoptimize_remapped_endpoints(const uint_vec &new_block_endpoints, int_vec &old_to_new_endpoint_cluster_indices, bool optimize_final_codebook, uint_vec *pBlock_selector_indices) { @@ -3002,12 +3003,12 @@ namespace basisu basisu::vector cluster_valid(new_endpoint_cluster_block_indices.size()); basisu::vector cluster_improved(new_endpoint_cluster_block_indices.size()); - + const uint32_t N = 256; for (uint32_t cluster_index_iter = 0; cluster_index_iter < new_endpoint_cluster_block_indices.size(); cluster_index_iter += N) { - const uint32_t first_index = cluster_index_iter; - const uint32_t last_index = minimum((uint32_t)new_endpoint_cluster_block_indices.size(), cluster_index_iter + N); + const uint32_t first_index = cluster_index_iter; + const uint32_t last_index = minimum((uint32_t)new_endpoint_cluster_block_indices.size(), cluster_index_iter + N); m_params.m_pJob_pool->add_job( [this, first_index, last_index, &cluster_improved, &cluster_valid, &new_endpoint_cluster_block_indices, &pBlock_selector_indices ] { @@ -3027,13 +3028,13 @@ namespace basisu blk.set_block_color5_etc1s(get_endpoint_cluster_unscaled_color(cluster_index, false)); blk.set_inten_tables_etc1s(get_endpoint_cluster_inten_table(cluster_index, false)); blk.set_flip_bit(true); - + uint64_t cur_err = 0; for (uint32_t cluster_block_indices_iter = 0; cluster_block_indices_iter < cluster_block_indices.size(); cluster_block_indices_iter++) { const uint32_t block_index = cluster_block_indices[cluster_block_indices_iter]; - + const color_rgba *pBlock_pixels = get_source_pixel_block(block_index).get_ptr(); memcpy(&cluster_pixels[cluster_block_indices_iter * 16], pBlock_pixels, 16 * sizeof(color_rgba)); @@ -3045,17 +3046,17 @@ namespace basisu blk.set_raw_selector_bits(blk_selectors.get_raw_selector_bits()); cur_err += blk.evaluate_etc1_error(pBlock_pixels, m_params.m_perceptual); - + for (uint32_t y = 0; y < 4; y++) for (uint32_t x = 0; x < 4; x++) force_selectors[cluster_block_indices_iter * 16 + x + y * 4] = static_cast(blk_selectors.get_selector(x, y)); } endpoint_cluster_etc_params new_endpoint_cluster_etc_params; - + { etc1_optimizer optimizer; - etc1_solution_coordinates solutions[2]; + //etc1_solution_coordinates solutions[2]; etc1_optimizer::params cluster_optimizer_params; cluster_optimizer_params.m_num_src_pixels = total_pixels; @@ -3065,7 +3066,7 @@ namespace basisu cluster_optimizer_params.m_perceptual = m_params.m_perceptual; cluster_optimizer_params.m_pForce_selectors = &force_selectors[0]; - if (m_params.m_compression_level == BASISU_MAX_COMPRESSION_LEVEL) + if (m_params.m_compression_level == BASISU_MAX_ETC1S_COMPRESSION_LEVEL) cluster_optimizer_params.m_quality = cETCQualityUber; else cluster_optimizer_params.m_quality = cETCQualitySlow; @@ -3091,7 +3092,7 @@ namespace basisu if (new_endpoint_cluster_etc_params.m_color_error[0] < cur_err) { m_endpoint_cluster_etc_params[cluster_index] = new_endpoint_cluster_etc_params; - + cluster_improved[cluster_index] = true; } @@ -3104,13 +3105,13 @@ namespace basisu } // cluster_index_iter m_params.m_pJob_pool->wait_for_all(); - + uint32_t total_unused_clusters = 0; uint32_t total_improved_clusters = 0; - + old_to_new_endpoint_cluster_indices.resize(m_endpoint_clusters.size()); vector_set_all(old_to_new_endpoint_cluster_indices, -1); - + int total_new_endpoint_clusters = 0; for (uint32_t old_cluster_index = 0; old_cluster_index < m_endpoint_clusters.size(); old_cluster_index++) @@ -3145,7 +3146,7 @@ namespace basisu for (uint32_t block_index = 0; block_index < new_block_endpoints.size(); block_index++) { const uint32_t old_endpoint_cluster_index = new_block_endpoints[block_index]; - + const int new_endpoint_cluster_index = old_to_new_endpoint_cluster_indices[old_endpoint_cluster_index]; BASISU_FRONTEND_VERIFY(new_endpoint_cluster_index >= 0); @@ -3158,13 +3159,13 @@ namespace basisu new_endpoint_cluster_etc_params[new_endpoint_cluster_index].m_subblocks.push_back(block_index * 2 + 0); new_endpoint_cluster_etc_params[new_endpoint_cluster_index].m_subblocks.push_back(block_index * 2 + 1); - + m_block_endpoint_clusters_indices[block_index][0] = new_endpoint_cluster_index; m_block_endpoint_clusters_indices[block_index][1] = new_endpoint_cluster_index; } debug_printf("basisu_frontend::reoptimize_remapped_endpoints: stage 2\n"); - + m_endpoint_clusters = new_endpoint_clusters; m_endpoint_cluster_etc_params = new_endpoint_cluster_etc_params; @@ -3200,7 +3201,7 @@ namespace basisu debug_printf("Final (post-RDO) endpoint clusters: %u\n", m_endpoint_clusters.size()); } - + //debug_printf("validate_output: %u\n", validate_output()); } @@ -3228,7 +3229,7 @@ namespace basisu // If the endpoint cluster lives in more than one parent node, that's wrong. if (subblock_parent_indices[subblock_index] != -1) return false; - + subblock_parent_indices[subblock_index] = parent_index; } } @@ -3252,7 +3253,7 @@ namespace basisu if (subblock_cluster_indices[subblock_index] != -1) return false; - + subblock_cluster_indices[subblock_index] = cluster_index; // There are transformations on the endpoint clusters that can break the strict tree requirement @@ -3266,7 +3267,7 @@ namespace basisu } } } - + // Make sure all endpoint clusters are present in the parent cluster. for (uint32_t i = 0; i < subblock_cluster_indices.size(); i++) { @@ -3291,7 +3292,7 @@ namespace basisu #define CHECK(x) BASISU_FRONTEND_VERIFY(x); CHECK(get_output_block(block_index).get_flip_bit() == true); - + const bool diff_flag = get_diff_flag(block_index); CHECK(diff_flag == true); @@ -3305,11 +3306,11 @@ namespace basisu // basisu only supports ETC1S, so these must be equal. CHECK(endpoint_cluster0_index == endpoint_cluster1_index); - + CHECK(blk.set_block_color5_check(get_endpoint_cluster_unscaled_color(endpoint_cluster0_index, false), get_endpoint_cluster_unscaled_color(endpoint_cluster1_index, false))); CHECK(get_endpoint_cluster_color_is_used(endpoint_cluster0_index, false)); - + blk.set_inten_table(0, get_endpoint_cluster_inten_table(endpoint_cluster0_index, false)); blk.set_inten_table(1, get_endpoint_cluster_inten_table(endpoint_cluster1_index, false)); @@ -3329,7 +3330,7 @@ namespace basisu CHECK(rdo_output_block.get_base5_color() == blk.get_base5_color()); CHECK(rdo_output_block.get_delta3_color() == blk.get_delta3_color()); CHECK(rdo_output_block.get_raw_selector_bits() == blk.get_raw_selector_bits()); - + #undef CHECK } @@ -3376,9 +3377,10 @@ namespace basisu } image img; - g.unpack(img); + g.unpack(img, false); save_png(pFilename, img); } } // namespace basisu + diff --git a/encoder/basisu_frontend.h b/encoder/basisu_frontend.h index 18ff5b6..a5aadb3 100644 --- a/encoder/basisu_frontend.h +++ b/encoder/basisu_frontend.h @@ -1,5 +1,5 @@ // basisu_frontend.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -37,8 +37,10 @@ namespace basisu uint32_t &operator[] (uint32_t i) { assert(i < 2); return m_comps[i]; } }; - const uint32_t BASISU_DEFAULT_COMPRESSION_LEVEL = 2; - const uint32_t BASISU_MAX_COMPRESSION_LEVEL = 6; + // rg [11/25/25] - The command line tool defaults to ETC1S level 1, but the API 2. Changing this breaks backwards compatibility for anyone using the API and our test suite. + const uint32_t BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL = 2; + + const uint32_t BASISU_MAX_ETC1S_COMPRESSION_LEVEL = 6; class basisu_frontend { @@ -61,7 +63,7 @@ namespace basisu enum { cMaxEndpointClusters = 16128, - + cMaxSelectorClusters = 16128, }; @@ -72,7 +74,7 @@ namespace basisu m_pSource_blocks(NULL), m_max_endpoint_clusters(256), m_max_selector_clusters(256), - m_compression_level(BASISU_DEFAULT_COMPRESSION_LEVEL), + m_compression_level(BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL), m_perceptual(true), m_debug_stats(false), m_debug_images(false), @@ -101,12 +103,12 @@ namespace basisu bool m_validate; bool m_multithreaded; bool m_disable_hierarchical_endpoint_codebooks; - + basist::basis_texture_type m_tex_type; const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks; - + opencl_context_ptr m_pOpenCL_context; - + job_pool *m_pJob_pool; }; @@ -143,12 +145,12 @@ namespace basisu uint32_t get_total_selector_clusters() const { return static_cast(m_selector_cluster_block_indices.size()); } uint32_t get_block_selector_cluster_index(uint32_t block_index) const { return m_block_selector_cluster_index[block_index]; } const etc_block &get_selector_cluster_selector_bits(uint32_t cluster_index) const { return m_optimized_cluster_selectors[cluster_index]; } - + // Returns block indices using each selector cluster const uint_vec &get_selector_cluster_block_indices(uint32_t selector_cluster_index) const { return m_selector_cluster_block_indices[selector_cluster_index]; } void dump_debug_image(const char *pFilename, uint32_t first_block, uint32_t num_blocks_x, uint32_t num_blocks_y, bool output_blocks); - + void reoptimize_remapped_endpoints(const uint_vec &new_block_endpoints, int_vec &old_to_new_endpoint_cluster_indices, bool optimize_final_codebook, uint_vec *pBlock_selector_indices = nullptr); bool get_opencl_failed() const { return m_opencl_failed; } @@ -170,15 +172,15 @@ namespace basisu // The quantized ETC1S texture. etc_block_vec m_encoded_blocks; - + // Quantized blocks after endpoint quant, but before selector quant - etc_block_vec m_orig_encoded_blocks; - + etc_block_vec m_orig_encoded_blocks; + // Full quality ETC1S texture etc_block_vec m_etc1_blocks_etc1s; - + typedef vec<6, float> vec6F; - + // Endpoint clusterizer typedef tree_vector_quant vec6F_quantizer; vec6F_quantizer m_endpoint_clusterizer; @@ -187,16 +189,16 @@ namespace basisu basisu::vector m_endpoint_clusters; // Array of subblock indices for each parent endpoint cluster - // Note: Initially, each endpoint cluster will only live in a single parent cluster, in a shallow tree. + // Note: Initially, each endpoint cluster will only live in a single parent cluster, in a shallow tree. // As the endpoint clusters are manipulated this constraint gets broken. basisu::vector m_endpoint_parent_clusters; - + // Each block's parent endpoint cluster index - uint8_vec m_block_parent_endpoint_cluster; + uint8_vec m_block_parent_endpoint_cluster; // Array of endpoint cluster indices for each parent endpoint cluster basisu::vector m_endpoint_clusters_within_each_parent_cluster; - + struct endpoint_cluster_etc_params { endpoint_cluster_etc_params() @@ -266,13 +268,13 @@ namespace basisu }; typedef basisu::vector cluster_subblock_etc_params_vec; - - // Each endpoint cluster's ETC1S parameters + + // Each endpoint cluster's ETC1S parameters cluster_subblock_etc_params_vec m_endpoint_cluster_etc_params; // The endpoint cluster index used by each ETC1 subblock. basisu::vector m_block_endpoint_clusters_indices; - + // The block(s) within each selector cluster // Note: If you add anything here that uses selector cluster indicies, be sure to update optimize_selector_codebook()! basisu::vector m_selector_cluster_block_indices; @@ -282,13 +284,13 @@ namespace basisu // The block(s) within each parent selector cluster. basisu::vector m_selector_parent_cluster_block_indices; - + // Each block's parent selector cluster uint8_vec m_block_parent_selector_cluster; // Array of selector cluster indices for each parent selector cluster basisu::vector m_selector_clusters_within_each_parent_cluster; - + // Each block's selector cluster index basisu::vector m_block_selector_cluster_index; diff --git a/encoder/basisu_gpu_texture.cpp b/encoder/basisu_gpu_texture.cpp index 983d6a8..59a2a17 100644 --- a/encoder/basisu_gpu_texture.cpp +++ b/encoder/basisu_gpu_texture.cpp @@ -1,5 +1,5 @@ // basisu_gpu_texture.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ namespace basisu //------------------------------------------------------------------------------------------------ // ETC2 EAC - void unpack_etc2_eac(const void *pBlock_bits, color_rgba *pPixels) + void unpack_etc2_eac(const void* pBlock_bits, color_rgba* pPixels) { static_assert(sizeof(eac_a8_block) == 8, "sizeof(eac_a8_block) == 8"); - const eac_a8_block *pBlock = static_cast(pBlock_bits); + const eac_a8_block* pBlock = static_cast(pBlock_bits); - const int8_t *pTable = g_etc2_eac_tables[pBlock->m_table]; + const int8_t* pTable = g_etc2_eac_tables[pBlock->m_table]; const uint64_t selector_bits = pBlock->get_selector_bits(); @@ -73,10 +73,10 @@ namespace basisu uint8_t m_high_color[cTotalEndpointBytes]; uint8_t m_selectors[cTotalSelectorBytes]; - inline uint32_t get_high_color() const { return m_high_color[0] | (m_high_color[1] << 8U); } + inline uint32_t get_high_color() const { return m_high_color[0] | (m_high_color[1] << 8U); } inline uint32_t get_low_color() const { return m_low_color[0] | (m_low_color[1] << 8U); } - static void unpack_color(uint32_t c, uint32_t &r, uint32_t &g, uint32_t &b) + static void unpack_color(uint32_t c, uint32_t& r, uint32_t& g, uint32_t& b) { r = (c >> 11) & 31; g = (c >> 5) & 63; @@ -91,11 +91,11 @@ namespace basisu }; // Returns true if the block uses 3 color punchthrough alpha mode. - bool unpack_bc1(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { static_assert(sizeof(bc1_block) == 8, "sizeof(bc1_block) == 8"); - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -147,11 +147,11 @@ namespace basisu return used_punchthrough; } - bool unpack_bc1_nv(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1_nv(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { static_assert(sizeof(bc1_block) == 8, "sizeof(bc1_block) == 8"); - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -182,19 +182,19 @@ namespace basisu if (l > h) { c[2].r = (uint8_t)(((2 * r0 + r1) * 22) / 8); - c[2].g = (uint8_t)(((256 * c[0].g + gdiff/4 + 128 + gdiff * 80) / 256)); + c[2].g = (uint8_t)(((256 * c[0].g + gdiff / 4 + 128 + gdiff * 80) / 256)); c[2].b = (uint8_t)(((2 * b0 + b1) * 22) / 8); c[2].a = 0xFF; c[3].r = (uint8_t)(((2 * r1 + r0) * 22) / 8); - c[3].g = (uint8_t)((256 * c[1].g - gdiff/4 + 128 - gdiff * 80) / 256); + c[3].g = (uint8_t)((256 * c[1].g - gdiff / 4 + 128 - gdiff * 80) / 256); c[3].b = (uint8_t)(((2 * b1 + b0) * 22) / 8); c[3].a = 0xFF; } else { c[2].r = (uint8_t)(((r0 + r1) * 33) / 8); - c[2].g = (uint8_t)((256 * c[0].g + gdiff/4 + 128 + gdiff * 128) / 256); + c[2].g = (uint8_t)((256 * c[0].g + gdiff / 4 + 128 + gdiff * 128) / 256); c[2].b = (uint8_t)(((b0 + b1) * 33) / 8); c[2].a = 0xFF; @@ -229,9 +229,9 @@ namespace basisu static inline int interp_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 * 43 + c1 * 21 + 32) >> 6; } static inline int interp_half_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 + c1 + 1) >> 1; } - bool unpack_bc1_amd(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha) + bool unpack_bc1_amd(const void* pBlock_bits, color_rgba* pPixels, bool set_alpha) { - const bc1_block *pBlock = static_cast(pBlock_bits); + const bc1_block* pBlock = static_cast(pBlock_bits); const uint32_t l = pBlock->get_low_color(); const uint32_t h = pBlock->get_high_color(); @@ -310,7 +310,7 @@ namespace basisu return (selector_bits >> (((y * 4) + x) * cBC4SelectorBits)) & (cMaxSelectorValues - 1); } - static inline uint32_t get_block_values6(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values6(uint8_t* pDst, uint32_t l, uint32_t h) { pDst[0] = static_cast(l); pDst[1] = static_cast(h); @@ -323,7 +323,7 @@ namespace basisu return 6; } - static inline uint32_t get_block_values8(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values8(uint8_t* pDst, uint32_t l, uint32_t h) { pDst[0] = static_cast(l); pDst[1] = static_cast(h); @@ -336,7 +336,7 @@ namespace basisu return 8; } - static inline uint32_t get_block_values(uint8_t *pDst, uint32_t l, uint32_t h) + static inline uint32_t get_block_values(uint8_t* pDst, uint32_t l, uint32_t h) { if (l > h) return get_block_values8(pDst, l, h); @@ -345,11 +345,11 @@ namespace basisu } }; - void unpack_bc4(const void *pBlock_bits, uint8_t *pPixels, uint32_t stride) + void unpack_bc4(const void* pBlock_bits, uint8_t* pPixels, uint32_t stride) { static_assert(sizeof(bc4_block) == 8, "sizeof(bc4_block) == 8"); - const bc4_block *pBlock = static_cast(pBlock_bits); + const bc4_block* pBlock = static_cast(pBlock_bits); uint8_t sel_values[8]; bc4_block::get_block_values(sel_values, pBlock->get_low_alpha(), pBlock->get_high_alpha()); @@ -366,11 +366,11 @@ namespace basisu } // Returns false if the block uses 3-color punchthrough alpha mode, which isn't supported on some GPU's for BC3. - bool unpack_bc3(const void *pBlock_bits, color_rgba *pPixels) + bool unpack_bc3(const void* pBlock_bits, color_rgba* pPixels) { bool success = true; - if (unpack_bc1((const uint8_t *)pBlock_bits + sizeof(bc4_block), pPixels, true)) + if (unpack_bc1((const uint8_t*)pBlock_bits + sizeof(bc4_block), pPixels, true)) success = false; unpack_bc4(pBlock_bits, &pPixels[0].a, sizeof(color_rgba)); @@ -379,10 +379,10 @@ namespace basisu } // writes RG - void unpack_bc5(const void *pBlock_bits, color_rgba *pPixels) + void unpack_bc5(const void* pBlock_bits, color_rgba* pPixels) { unpack_bc4(pBlock_bits, &pPixels[0].r, sizeof(color_rgba)); - unpack_bc4((const uint8_t *)pBlock_bits + sizeof(bc4_block), &pPixels[0].g, sizeof(color_rgba)); + unpack_bc4((const uint8_t*)pBlock_bits + sizeof(bc4_block), &pPixels[0].g, sizeof(color_rgba)); } //------------------------------------------------------------------------------------------------ @@ -439,323 +439,6 @@ namespace basisu } } - //------------------------------------------------------------------------------------------------ - // BC7 mode 0-7 decompression. - // Instead of one monster routine to unpack all the BC7 modes, we're lumping the 3 subset, 2 subset, 1 subset, and dual plane modes together into simple shared routines. - - static inline uint32_t bc7_dequant(uint32_t val, uint32_t pbit, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(pbit < 2); assert(val_bits >= 4 && val_bits <= 8); const uint32_t total_bits = val_bits + 1; val = (val << 1) | pbit; val <<= (8 - total_bits); val |= (val >> total_bits); assert(val <= 255); return val; } - static inline uint32_t bc7_dequant(uint32_t val, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(val_bits >= 4 && val_bits <= 8); val <<= (8 - val_bits); val |= (val >> val_bits); assert(val <= 255); return val; } - - static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w < 4); return (l * (64 - basist::g_bc7_weights2[w]) + h * basist::g_bc7_weights2[w] + 32) >> 6; } - static inline uint32_t bc7_interp3(uint32_t l, uint32_t h, uint32_t w) { assert(w < 8); return (l * (64 - basist::g_bc7_weights3[w]) + h * basist::g_bc7_weights3[w] + 32) >> 6; } - static inline uint32_t bc7_interp4(uint32_t l, uint32_t h, uint32_t w) { assert(w < 16); return (l * (64 - basist::g_bc7_weights4[w]) + h * basist::g_bc7_weights4[w] + 32) >> 6; } - static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w, uint32_t bits) - { - assert(l <= 255 && h <= 255); - switch (bits) - { - case 2: return bc7_interp2(l, h, w); - case 3: return bc7_interp3(l, h, w); - case 4: return bc7_interp4(l, h, w); - default: - break; - } - return 0; - } - - bool unpack_bc7_mode0_2(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - //const uint32_t SUBSETS = 3; - const uint32_t ENDPOINTS = 6; - const uint32_t COMPS = 3; - const uint32_t WEIGHT_BITS = (mode == 0) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 0) ? 4 : 5; - const uint32_t PBITS = (mode == 0) ? 6 : 0; - const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t part = read_bits32(pBuf, bit_offset, (mode == 0) ? 4 : 6); - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); - - uint32_t pbits[6]; - for (uint32_t p = 0; p < PBITS; p++) - pbits[p] = read_bits32(pBuf, bit_offset, 1); - - uint32_t weights[16]; - for (uint32_t i = 0; i < 16; i++) - weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_third_subset_1[part]) || (i == basist::g_bc7_table_anchor_index_third_subset_2[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)((c == 3) ? 255 : (PBITS ? bc7_dequant(endpoints[e][c], pbits[e], ENDPOINT_BITS) : bc7_dequant(endpoints[e][c], ENDPOINT_BITS))); - - color_rgba block_colors[3][8]; - for (uint32_t s = 0; s < 3; s++) - for (uint32_t i = 0; i < WEIGHT_VALS; i++) - { - for (uint32_t c = 0; c < 3; c++) - block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); - block_colors[s][i][3] = 255; - } - - for (uint32_t i = 0; i < 16; i++) - pPixels[i] = block_colors[basist::g_bc7_partition3[part * 16 + i]][weights[i]]; - - return true; - } - - bool unpack_bc7_mode1_3_7(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - //const uint32_t SUBSETS = 2; - const uint32_t ENDPOINTS = 4; - const uint32_t COMPS = (mode == 7) ? 4 : 3; - const uint32_t WEIGHT_BITS = (mode == 1) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 7) ? 5 : ((mode == 1) ? 6 : 7); - const uint32_t PBITS = (mode == 1) ? 2 : 4; - const uint32_t SHARED_PBITS = (mode == 1) ? true : false; - const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t part = read_bits32(pBuf, bit_offset, 6); - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); - - uint32_t pbits[4]; - for (uint32_t p = 0; p < PBITS; p++) - pbits[p] = read_bits32(pBuf, bit_offset, 1); - - uint32_t weights[16]; - for (uint32_t i = 0; i < 16; i++) - weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_second_subset[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)((c == ((mode == 7U) ? 4U : 3U)) ? 255 : bc7_dequant(endpoints[e][c], pbits[SHARED_PBITS ? (e >> 1) : e], ENDPOINT_BITS)); - - color_rgba block_colors[2][8]; - for (uint32_t s = 0; s < 2; s++) - for (uint32_t i = 0; i < WEIGHT_VALS; i++) - { - for (uint32_t c = 0; c < COMPS; c++) - block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); - block_colors[s][i][3] = (COMPS == 3) ? 255 : block_colors[s][i][3]; - } - - for (uint32_t i = 0; i < 16; i++) - pPixels[i] = block_colors[basist::g_bc7_partition2[part * 16 + i]][weights[i]]; - - return true; - } - - bool unpack_bc7_mode4_5(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) - { - const uint32_t ENDPOINTS = 2; - const uint32_t COMPS = 4; - const uint32_t WEIGHT_BITS = 2; - const uint32_t A_WEIGHT_BITS = (mode == 4) ? 3 : 2; - const uint32_t ENDPOINT_BITS = (mode == 4) ? 5 : 7; - const uint32_t A_ENDPOINT_BITS = (mode == 4) ? 6 : 8; - //const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; - //const uint32_t A_WEIGHT_VALS = 1 << A_WEIGHT_BITS; - - uint32_t bit_offset = 0; - const uint8_t* pBuf = static_cast(pBlock_bits); - - if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; - - const uint32_t comp_rot = read_bits32(pBuf, bit_offset, 2); - const uint32_t index_mode = (mode == 4) ? read_bits32(pBuf, bit_offset, 1) : 0; - - color_rgba endpoints[ENDPOINTS]; - for (uint32_t c = 0; c < COMPS; c++) - for (uint32_t e = 0; e < ENDPOINTS; e++) - endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); - - const uint32_t weight_bits[2] = { index_mode ? A_WEIGHT_BITS : WEIGHT_BITS, index_mode ? WEIGHT_BITS : A_WEIGHT_BITS }; - - uint32_t weights[16], a_weights[16]; - - for (uint32_t i = 0; i < 16; i++) - (index_mode ? a_weights : weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[index_mode] - ((!i) ? 1 : 0)); - - for (uint32_t i = 0; i < 16; i++) - (index_mode ? weights : a_weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[1 - index_mode] - ((!i) ? 1 : 0)); - - assert(bit_offset == 128); - - for (uint32_t e = 0; e < ENDPOINTS; e++) - for (uint32_t c = 0; c < 4; c++) - endpoints[e][c] = (uint8_t)bc7_dequant(endpoints[e][c], (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); - - color_rgba block_colors[8]; - for (uint32_t i = 0; i < (1U << weight_bits[0]); i++) - for (uint32_t c = 0; c < 3; c++) - block_colors[i][c] = (uint8_t)bc7_interp(endpoints[0][c], endpoints[1][c], i, weight_bits[0]); - - for (uint32_t i = 0; i < (1U << weight_bits[1]); i++) - block_colors[i][3] = (uint8_t)bc7_interp(endpoints[0][3], endpoints[1][3], i, weight_bits[1]); - - for (uint32_t i = 0; i < 16; i++) - { - pPixels[i] = block_colors[weights[i]]; - pPixels[i].a = block_colors[a_weights[i]].a; - if (comp_rot >= 1) - std::swap(pPixels[i].a, pPixels[i].m_comps[comp_rot - 1]); - } - - return true; - } - - struct bc7_mode_6 - { - struct - { - uint64_t m_mode : 7; - uint64_t m_r0 : 7; - uint64_t m_r1 : 7; - uint64_t m_g0 : 7; - uint64_t m_g1 : 7; - uint64_t m_b0 : 7; - uint64_t m_b1 : 7; - uint64_t m_a0 : 7; - uint64_t m_a1 : 7; - uint64_t m_p0 : 1; - } m_lo; - - union - { - struct - { - uint64_t m_p1 : 1; - uint64_t m_s00 : 3; - uint64_t m_s10 : 4; - uint64_t m_s20 : 4; - uint64_t m_s30 : 4; - - uint64_t m_s01 : 4; - uint64_t m_s11 : 4; - uint64_t m_s21 : 4; - uint64_t m_s31 : 4; - - uint64_t m_s02 : 4; - uint64_t m_s12 : 4; - uint64_t m_s22 : 4; - uint64_t m_s32 : 4; - - uint64_t m_s03 : 4; - uint64_t m_s13 : 4; - uint64_t m_s23 : 4; - uint64_t m_s33 : 4; - - } m_hi; - - uint64_t m_hi_bits; - }; - }; - - bool unpack_bc7_mode6(const void *pBlock_bits, color_rgba *pPixels) - { - static_assert(sizeof(bc7_mode_6) == 16, "sizeof(bc7_mode_6) == 16"); - - const bc7_mode_6 &block = *static_cast(pBlock_bits); - - if (block.m_lo.m_mode != (1 << 6)) - return false; - - const uint32_t r0 = (uint32_t)((block.m_lo.m_r0 << 1) | block.m_lo.m_p0); - const uint32_t g0 = (uint32_t)((block.m_lo.m_g0 << 1) | block.m_lo.m_p0); - const uint32_t b0 = (uint32_t)((block.m_lo.m_b0 << 1) | block.m_lo.m_p0); - const uint32_t a0 = (uint32_t)((block.m_lo.m_a0 << 1) | block.m_lo.m_p0); - const uint32_t r1 = (uint32_t)((block.m_lo.m_r1 << 1) | block.m_hi.m_p1); - const uint32_t g1 = (uint32_t)((block.m_lo.m_g1 << 1) | block.m_hi.m_p1); - const uint32_t b1 = (uint32_t)((block.m_lo.m_b1 << 1) | block.m_hi.m_p1); - const uint32_t a1 = (uint32_t)((block.m_lo.m_a1 << 1) | block.m_hi.m_p1); - - color_rgba vals[16]; - for (uint32_t i = 0; i < 16; i++) - { - const uint32_t w = basist::g_bc7_weights4[i]; - const uint32_t iw = 64 - w; - vals[i].set_noclamp_rgba( - (r0 * iw + r1 * w + 32) >> 6, - (g0 * iw + g1 * w + 32) >> 6, - (b0 * iw + b1 * w + 32) >> 6, - (a0 * iw + a1 * w + 32) >> 6); - } - - pPixels[0] = vals[block.m_hi.m_s00]; - pPixels[1] = vals[block.m_hi.m_s10]; - pPixels[2] = vals[block.m_hi.m_s20]; - pPixels[3] = vals[block.m_hi.m_s30]; - - pPixels[4] = vals[block.m_hi.m_s01]; - pPixels[5] = vals[block.m_hi.m_s11]; - pPixels[6] = vals[block.m_hi.m_s21]; - pPixels[7] = vals[block.m_hi.m_s31]; - - pPixels[8] = vals[block.m_hi.m_s02]; - pPixels[9] = vals[block.m_hi.m_s12]; - pPixels[10] = vals[block.m_hi.m_s22]; - pPixels[11] = vals[block.m_hi.m_s32]; - - pPixels[12] = vals[block.m_hi.m_s03]; - pPixels[13] = vals[block.m_hi.m_s13]; - pPixels[14] = vals[block.m_hi.m_s23]; - pPixels[15] = vals[block.m_hi.m_s33]; - - return true; - } - - bool unpack_bc7(const void *pBlock, color_rgba *pPixels) - { - const uint32_t first_byte = static_cast(pBlock)[0]; - - for (uint32_t mode = 0; mode <= 7; mode++) - { - if (first_byte & (1U << mode)) - { - switch (mode) - { - case 0: - case 2: - return unpack_bc7_mode0_2(mode, pBlock, pPixels); - case 1: - case 3: - case 7: - return unpack_bc7_mode1_3_7(mode, pBlock, pPixels); - case 4: - case 5: - return unpack_bc7_mode4_5(mode, pBlock, pPixels); - case 6: - return unpack_bc7_mode6(pBlock, pPixels); - default: - break; - } - } - } - - return false; - } - static inline int bc6h_sign_extend(int val, int bits) { assert((bits >= 1) && (bits < 32)); @@ -1105,7 +788,7 @@ namespace basisu return false; if (pBlock->m_hi.m_alpha == 1) return false; - + color_rgba colors[4]; colors[0].r = pBlock->m_hi.m_r0; @@ -1155,7 +838,7 @@ namespace basisu for (uint32_t i = 0; i < 16; i++) { const uint32_t sel = (pBlock->m_sels[4 + (i >> 2)] >> ((i & 3) * 2)) & 3; - + const uint32_t x = i & 3; const uint32_t y = i >> 2; pPixels[4 + x + y * 8] = block1_colors[sel]; @@ -1216,7 +899,7 @@ namespace basisu { return color_rgba((col[0] << 3) | (col[0] >> 2), (col[1] << 3) | (col[1] >> 2), (col[2] << 3) | (col[2] >> 2), 255); } - + static color_rgba convert_rgba_5554_to_8888(const color_rgba& col) { return color_rgba((col[0] << 3) | (col[0] >> 2), (col[1] << 3) | (col[1] >> 2), (col[2] << 3) | (col[2] >> 2), (col[3] << 4) | col[3]); @@ -1239,10 +922,10 @@ namespace basisu { // colora=554 color_rgba color_a(pBlock->m_opaque_color_data.m_red_a, pBlock->m_opaque_color_data.m_green_a, (pBlock->m_opaque_color_data.m_blue_a << 1) | (pBlock->m_opaque_color_data.m_blue_a >> 3), 255); - + // colora=555 color_rgba color_b(pBlock->m_opaque_color_data.m_red_b, pBlock->m_opaque_color_data.m_green_b, pBlock->m_opaque_color_data.m_blue_b, 255); - + colors[0] = convert_rgb_555_to_888(color_a); colors[3] = convert_rgb_555_to_888(color_b); @@ -1251,11 +934,11 @@ namespace basisu } else { - // colora=4433 + // colora=4433 color_rgba color_a( - (pBlock->m_trans_color_data.m_red_a << 1) | (pBlock->m_trans_color_data.m_red_a >> 3), + (pBlock->m_trans_color_data.m_red_a << 1) | (pBlock->m_trans_color_data.m_red_a >> 3), (pBlock->m_trans_color_data.m_green_a << 1) | (pBlock->m_trans_color_data.m_green_a >> 3), - (pBlock->m_trans_color_data.m_blue_a << 2) | (pBlock->m_trans_color_data.m_blue_a >> 1), + (pBlock->m_trans_color_data.m_blue_a << 2) | (pBlock->m_trans_color_data.m_blue_a >> 1), pBlock->m_trans_color_data.m_alpha_a << 1); //colorb=4443 @@ -1331,9 +1014,9 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { const uint32_t shift = 45 - ((y + x * 4) * 3); - + const uint32_t sel = (uint32_t)((sels >> shift) & 7); - + int val = base + g_etc2_eac_tables[table][sel] * mul; val = clamp(val, 0, 2047); @@ -1362,9 +1045,10 @@ namespace basisu { basist::unpack_uastc(*static_cast(p), (basist::color32 *)pPixels, false); } - + // Unpacks to RGBA, R, RG, or A. LDR GPU texture formats only. - bool unpack_block(texture_format fmt, const void* pBlock, color_rgba* pPixels) + // astc_srgb: if true, ASTC LDR formats are decoded in sRGB decode mode, otherwise L8. + bool unpack_block(texture_format fmt, const void* pBlock, color_rgba* pPixels, bool astc_srgb) { switch (fmt) { @@ -1400,7 +1084,7 @@ namespace basisu } case texture_format::cBC7: { - return unpack_bc7(pBlock, pPixels); + return basist::bc7u::unpack_bc7(pBlock, reinterpret_cast(pPixels)); } // Full ETC2 color blocks (planar/T/H modes) is currently unsupported in basisu, but we do support ETC2 with alpha (using ETC1 for color) case texture_format::cETC2_RGB: @@ -1433,14 +1117,32 @@ namespace basisu return false; } case texture_format::cASTC_LDR_4x4: + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: { - const bool astc_srgb = false; - bool status = basisu_astc::astc::decompress_ldr(reinterpret_cast(pPixels), static_cast(pBlock), astc_srgb, 4, 4); + const uint32_t block_width = get_block_width(fmt), block_height = get_block_height(fmt); + + assert(get_astc_ldr_texture_format(block_width, block_height) == fmt); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); + + // TODO: Allow caller to use the Android decoder, too. + bool status = basisu_astc::astc::decompress_ldr(reinterpret_cast(pPixels), static_cast(pBlock), astc_srgb, block_width, block_height); assert(status); if (!status) return false; - + break; } case texture_format::cATC_RGB: @@ -1532,7 +1234,7 @@ namespace basisu #else // Use our decoder basist::half_float half_block[16][4]; - + astc_helpers::log_astc_block log_blk; if (!astc_helpers::unpack_block(pBlock, log_blk, 4, 4)) return false; @@ -1577,8 +1279,8 @@ namespace basisu assert(0); return false; } - - bool gpu_image::unpack(image& img) const + + bool gpu_image::unpack(image& img, bool astc_srgb) const { img.resize(get_pixel_width(), get_pixel_height()); img.set_all(g_black_color); @@ -1589,11 +1291,11 @@ namespace basisu if ((m_fmt == texture_format::cPVRTC1_4_RGB) || (m_fmt == texture_format::cPVRTC1_4_RGBA)) { pvrtc4_image pi(m_width, m_height); - + if (get_total_blocks() != pi.get_total_blocks()) return false; - - memcpy(&pi.get_blocks()[0], get_ptr(), get_size_in_bytes()); + + memcpy((void *)&pi.get_blocks()[0], (const void *)get_ptr(), get_size_in_bytes()); pi.deswizzle(); @@ -1615,7 +1317,7 @@ namespace basisu { const void* pBlock = get_block_ptr(bx, by); - if (!unpack_block(m_fmt, pBlock, pixels)) + if (!unpack_block(m_fmt, pBlock, pixels, astc_srgb)) success = false; img.set_block_clipped(pixels, bx * m_block_width, by * m_block_height, m_block_width, m_block_height); @@ -1662,14 +1364,14 @@ namespace basisu return success; } - + // KTX1 texture file writing static const uint8_t g_ktx_file_id[12] = { 0xAB, 0x4B, 0x54, 0x58, 0x20, 0x31, 0x31, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A }; // KTX/GL enums enum { - KTX_ENDIAN = 0x04030201, + KTX_ENDIAN = 0x04030201, KTX_OPPOSITE_ENDIAN = 0x01020304, KTX_ETC1_RGB8_OES = 0x8D64, KTX_RED = 0x1903, @@ -1689,7 +1391,7 @@ namespace basisu KTX_COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT = 0x8E8F, KTX_COMPRESSED_RGB_PVRTC_4BPPV1_IMG = 0x8C00, KTX_COMPRESSED_RGBA_PVRTC_4BPPV1_IMG = 0x8C02, - + KTX_COMPRESSED_RGBA_ASTC_4x4_KHR = 0x93B0, KTX_COMPRESSED_RGBA_ASTC_5x4_KHR = 0x93B1, KTX_COMPRESSED_RGBA_ASTC_5x5_KHR = 0x93B2, @@ -1731,7 +1433,7 @@ namespace basisu KTX_COMPRESSED_R11_EAC = 0x9270, KTX_COMPRESSED_RG11_EAC = 0x9272 }; - + struct ktx_header { uint8_t m_identifier[12]; @@ -1753,7 +1455,7 @@ namespace basisu }; // Input is a texture array of mipmapped gpu_image's: gpu_images[array_index][level_index] - bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag) + bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag, bool astc_srgb_flag) { if (!gpu_images.size()) { @@ -1773,7 +1475,7 @@ namespace basisu return false; } } - + for (uint32_t array_index = 0; array_index < gpu_images.size(); array_index++) { const gpu_image_vec &levels = gpu_images[array_index]; @@ -1905,18 +1607,101 @@ namespace basisu { internal_fmt = KTX_COMPRESSED_RGBA_ASTC_6x6_KHR; // TODO: should we write RGB? We don't support generating HDR 6x6 with alpha. - base_internal_fmt = KTX_RGBA; + base_internal_fmt = KTX_RGBA; break; } // We use different enums for HDR vs. LDR ASTC, but internally they are both just ASTC. - case texture_format::cASTC_LDR_4x4: case texture_format::cASTC_HDR_4x4: - case texture_format::cUASTC_HDR_4x4: // UASTC_HDR is just HDR-only ASTC + case texture_format::cUASTC_HDR_4x4: // UASTC_HDR 4x4 is just HDR-only ASTC { internal_fmt = KTX_COMPRESSED_RGBA_ASTC_4x4_KHR; base_internal_fmt = KTX_RGBA; break; } + case texture_format::cASTC_LDR_4x4: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_4x4_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_5x4: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_5x4_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_5x4_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_5x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_5x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_5x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_6x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_6x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_6x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_6x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_6x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_6x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x5: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x5_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x5_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x6: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x6_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x6_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_8x8: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_8x8_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_8x8_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x8: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x8_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x8_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_10x10: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_10x10_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_10x10_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_12x10: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_12x10_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_12x10_KHR; + base_internal_fmt = KTX_RGBA; + break; + } + case texture_format::cASTC_LDR_12x12: + { + internal_fmt = !astc_srgb_flag ? KTX_COMPRESSED_RGBA_ASTC_12x12_KHR : KTX_COMPRESSED_SRGB8_ALPHA8_ASTC_12x12_KHR; + base_internal_fmt = KTX_RGBA; + break; + } case texture_format::cATC_RGB: { internal_fmt = KTX_ATC_RGB_AMD; @@ -1987,6 +1772,12 @@ namespace basisu append_vector(ktx_data, (uint8_t*)&header, sizeof(header)); + fmt_debug_printf("create_ktx_texture_file: {}x{}, astc_srgb_flag: {}, basis::texture_format: {}, internalFormat: {}, baseInternalFormat: {}, arrayElements: {}, faces: {}, mipLevels: {}\n", + width, height, astc_srgb_flag, (uint32_t)fmt, + (uint32_t)header.m_glInternalFormat, (uint32_t)header.m_glBaseInternalFormat, + (uint32_t)header.m_numberOfArrayElements, (uint32_t)header.m_numberOfFaces, + (uint32_t)header.m_numberOfMipmapLevels); + for (uint32_t level_index = 0; level_index < total_levels; level_index++) { uint32_t img_size = gpu_images[0][level_index].get_size_in_bytes(); @@ -2169,7 +1960,7 @@ namespace basisu } // array_index } #endif - + // Write DDS file using tinydds TinyDDS_WriteCallbacks cbs; cbs.error = [](void* user, char const* msg) { BASISU_NOTE_UNUSED(user); fprintf(stderr, "tinydds: %s\n", msg); }; @@ -2179,7 +1970,7 @@ namespace basisu uint32_t mipmap_sizes[32]; const void* mipmap_ptrs[32]; - + clear_obj(mipmap_sizes); clear_obj(mipmap_ptrs); @@ -2197,7 +1988,7 @@ namespace basisu { case texture_format::cBC1_NV: case texture_format::cBC1_AMD: - case texture_format::cBC1: + case texture_format::cBC1: tinydds_fmt = use_srgb_format ? TDDS_BC1_RGBA_SRGB_BLOCK : TDDS_BC1_RGBA_UNORM_BLOCK; break; case texture_format::cBC3: @@ -2225,10 +2016,10 @@ namespace basisu } } - // DirectXTex's DDSView doesn't handle odd sizes textures correctly. RenderDoc loads them fine, however. - // Trying to work around this here results in invalid mipmaps. - //width = (width + 3) & ~3; - //height = (height + 3) & ~3; + // Note DirectXTex's DDSView doesn't handle odd sizes textures correctly. RenderDoc loads them fine, however. + + fmt_debug_printf("write_dds_file: {}x{}, basis::texture_format: {}, tinydds_fmt: {}, slices: {}, mipLevels: {}, cubemap_flag: {}, use_srgb_format: {}\n", + width, height, (uint32_t)fmt, tinydds_fmt, slices, total_levels, cubemap_flag, use_srgb_format); bool status = TinyDDS_WriteImage(&cbs, &dds_data, @@ -2248,7 +2039,7 @@ namespace basisu fprintf(stderr, "write_dds_file: Failed creating DDS file\n"); return false; } - + return true; } @@ -2267,7 +2058,7 @@ namespace basisu return true; } - + bool read_uncompressed_dds_file(const char* pFilename, basisu::vector &ldr_mips, basisu::vector& hdr_mips) { const uint32_t MAX_IMAGE_DIM = 16384; @@ -2278,7 +2069,7 @@ namespace basisu cbs.allocFn = [](void* user, size_t size) -> void* { BASISU_NOTE_UNUSED(user); return malloc(size); }; cbs.freeFn = [](void* user, void* memory) { BASISU_NOTE_UNUSED(user); free(memory); }; cbs.readFn = [](void* user, void* buffer, size_t byteCount) -> size_t { return (size_t)fread(buffer, 1, byteCount, (FILE*)user); }; - + #ifdef _MSC_VER cbs.seekFn = [](void* user, int64_t ofs) -> bool { return _fseeki64((FILE*)user, ofs, SEEK_SET) == 0; }; cbs.tellFn = [](void* user) -> int64_t { return _ftelli64((FILE*)user); }; @@ -2318,7 +2109,7 @@ namespace basisu error_printf("Failed parsing DDS header in file \"%s\"\n", pFilename); goto failure; } - + if ((!TinyDDS_Is2D(ctx)) || (TinyDDS_ArraySlices(ctx) > 1) || (TinyDDS_IsCubemap(ctx))) { error_printf("Unsupported DDS texture type in file \"%s\"\n", pFilename); @@ -2327,7 +2118,7 @@ namespace basisu width = TinyDDS_Width(ctx); height = TinyDDS_Height(ctx); - + if (!width || !height) { error_printf("DDS texture dimensions invalid in file \"%s\"\n", pFilename); @@ -2339,7 +2130,7 @@ namespace basisu error_printf("DDS texture dimensions too large in file \"%s\"\n", pFilename); goto failure; } - + tfmt = TinyDDS_GetFormat(ctx); switch (tfmt) { @@ -2387,7 +2178,7 @@ namespace basisu } memcpy(ldr_mips[level].get_ptr(), pImage, image_size); - + if ((tfmt == TDDS_B8G8R8A8_SRGB) || (tfmt == TDDS_B8G8R8A8_UNORM)) { // Swap R and B components. @@ -2416,7 +2207,7 @@ namespace basisu else if (fmt == cRGBA_HALF) { hdr_mips[level].resize(level_width, level_height); - + if ((hdr_mips[level].get_total_pixels() * sizeof(basist::half_float) * 4 != image_size)) { assert(0); @@ -2426,7 +2217,7 @@ namespace basisu // Unpack half to float. const basist::half_float* pSrc_comps = static_cast(pImage); vec4F* pDst_texels = hdr_mips[level].get_ptr(); - + for (uint32_t i = 0; i < total_level_texels; i++) { (*pDst_texels)[0] = basist::half_to_float(pSrc_comps[0]); @@ -2462,7 +2253,7 @@ namespace basisu uint8_vec filedata; if (extension == "ktx") { - if (!create_ktx_texture_file(filedata, g, cubemap_flag)) + if (!create_ktx_texture_file(filedata, g, cubemap_flag, use_srgb_format)) return false; } else if (extension == "pvr") @@ -2500,7 +2291,7 @@ namespace basisu } //const uint32_t OUT_FILE_MAGIC = 'TEXC'; - struct out_file_header + struct out_file_header { packed_uint<4> m_magic; packed_uint<4> m_pad; @@ -2532,16 +2323,92 @@ namespace basisu fwrite(&hdr, sizeof(hdr), 1, pFile); fwrite(gi.get_ptr(), gi.get_size_in_bytes(), 1, pFile); - + return fclose(pFile) != EOF; } - // The .astc texture format is readable using ARM's astcenc, AMD Compressonator, and other engines/tools. It oddly doesn't support mipmaps, limiting +#pragma pack(push, 1) + struct astc_file_header + { + uint8_t m_sig[4]; + uint8_t m_block_dim[3]; + uint8_t m_width[3]; + uint8_t m_height[3]; + uint8_t m_depth[3]; + }; +#pragma pack(pop) + + bool read_astc_file(const uint8_t *pImage_data, size_t image_data_size, vector2D& blocks, uint32_t &block_width, uint32_t &block_height, uint32_t &width, uint32_t &height) + { + block_width = 0; + block_height = 0; + width = 0; + height = 0; + blocks.resize(0, 0); + + if (image_data_size < (sizeof(astc_file_header) + sizeof(astc_helpers::astc_block))) + return false; + + const astc_file_header* pHeader = reinterpret_cast(pImage_data); + + if ((pHeader->m_sig[0] != 0x13) || (pHeader->m_sig[1] != 0xAB) || (pHeader->m_sig[2] != 0xA1) || (pHeader->m_sig[3] != 0x5C)) + return false; + + const uint32_t block_depth = pHeader->m_block_dim[2]; + if (block_depth != 1) + return false; + + if ((pHeader->m_depth[0] != 1) || (pHeader->m_depth[1] != 0) || (pHeader->m_depth[2] != 0)) + return false; + + block_width = pHeader->m_block_dim[0]; + block_height = pHeader->m_block_dim[1]; + + if (!astc_helpers::is_valid_block_size(block_width, block_height)) + return false; + + width = pHeader->m_width[0] | ((uint32_t)pHeader->m_width[1] << 8u) | ((uint32_t)pHeader->m_width[2] << 16u); + height = pHeader->m_height[0] | ((uint32_t)pHeader->m_height[1] << 8u) | ((uint32_t)pHeader->m_height[2] << 16u); + + const uint32_t MAX_DIM = 32768; + if ((!width) || (width > MAX_DIM) || (!height) || (height > MAX_DIM)) + return false; + + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + const uint32_t total_blocks = num_blocks_x * num_blocks_y; + + size_t total_expected_size = sizeof(astc_file_header) + (size_t)total_blocks * sizeof(astc_helpers::astc_block); + if (image_data_size < total_expected_size) + return false; + + if (!blocks.try_resize(num_blocks_x, num_blocks_y)) + return false; + + memcpy(blocks.get_ptr(), pImage_data + sizeof(astc_file_header), (size_t)total_blocks * sizeof(astc_helpers::astc_block)); + + return true; + } + + bool read_astc_file(const char* pFilename, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height) + { + uint8_vec file_data; + if (!read_file_to_vec(pFilename, file_data)) + return false; + + if (!file_data.size()) + return false; + + return read_astc_file(file_data.get_ptr(), file_data.size(), blocks, block_width, block_height, width, height); + } + + // The .astc texture format is readable using ARM's astcenc, AMD Compressonator, and other engines/tools. It oddly doesn't support mipmaps, limiting // its usefulness/relevance. // https://github.com/ARM-software/astc-encoder/blob/main/Docs/FileFormat.md bool write_astc_file(const char* pFilename, const void* pBlocks, uint32_t block_width, uint32_t block_height, uint32_t dim_x, uint32_t dim_y) { - assert(pBlocks && (block_width >= 4) && (block_height >= 4) && (dim_x > 0) && (dim_y > 0)); + assert(pBlocks && (dim_x > 0) && (dim_y > 0)); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); uint8_vec file_data; file_data.push_back(0x13); @@ -2578,5 +2445,6 @@ namespace basisu return write_vec_to_file(pFilename, file_data); } - + } // basisu + diff --git a/encoder/basisu_gpu_texture.h b/encoder/basisu_gpu_texture.h index 06f2cc0..bcfc9cb 100644 --- a/encoder/basisu_gpu_texture.h +++ b/encoder/basisu_gpu_texture.h @@ -1,5 +1,5 @@ // basisu_gpu_texture.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ // limitations under the License. #pragma once #include "../transcoder/basisu.h" +#include "../transcoder/basisu_astc_helpers.h" #include "basisu_etc.h" namespace basisu @@ -49,11 +50,12 @@ namespace basisu inline texture_format get_format() const { return m_fmt; } inline bool is_hdr() const { return is_hdr_texture_format(m_fmt); } - + inline bool is_ldr() const { return !is_hdr_texture_format(m_fmt); } + // Width/height in pixels inline uint32_t get_pixel_width() const { return m_width; } inline uint32_t get_pixel_height() const { return m_height; } - + // Width/height in blocks, row pitch is assumed to be m_blocks_x. inline uint32_t get_blocks_x() const { return m_blocks_x; } inline uint32_t get_blocks_y() const { return m_blocks_y; } @@ -68,7 +70,7 @@ namespace basisu inline uint32_t get_row_pitch_in_bytes() const { return get_bytes_per_block() * get_blocks_x(); } inline const uint64_vec &get_blocks() const { return m_blocks; } - + inline const uint64_t *get_ptr() const { return &m_blocks[0]; } inline uint64_t *get_ptr() { return &m_blocks[0]; } @@ -101,12 +103,14 @@ namespace basisu m_blocks.resize(m_blocks_x * m_blocks_y * m_qwords_per_block); } - // Unpacks LDR textures only. - bool unpack(image& img) const; + // Unpacks LDR textures only. Asserts and returns false otherwise. + // astc_srgb: true to use the ASTC sRGB decode profile, false for linear. + // For XUASTC LDR, this should match what was used during encoding. For ETC1S/UASTC LDR 4x4, this should be false. + bool unpack(image& img, bool astc_srgb) const; - // Unpacks HDR textures only. + // Unpacks HDR textures only. Asserts and returns false otherwise. bool unpack_hdr(imagef& img) const; - + inline void override_dimensions(uint32_t w, uint32_t h) { m_width = w; @@ -121,9 +125,10 @@ namespace basisu typedef basisu::vector gpu_image_vec; - // KTX1 file writing - bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag); - + // KTX1 file writing - compatible with ARM's astcenc tool, and some other tools. + // Note astc_linear_flag used to be always effectively true in older code. It's ignored for ASTC HDR formats. + bool create_ktx_texture_file(uint8_vec &ktx_data, const basisu::vector& gpu_images, bool cubemap_flag, bool astc_srgb_flag); + bool does_dds_support_format(texture_format fmt); bool write_dds_file(uint8_vec& dds_data, const basisu::vector& gpu_images, bool cubemap_flag, bool use_srgb_format); bool write_dds_file(const char* pFilename, const basisu::vector& gpu_images, bool cubemap_flag, bool use_srgb_format); @@ -135,7 +140,7 @@ namespace basisu bool write_compressed_texture_file(const char *pFilename, const basisu::vector& g, bool cubemap_flag, bool use_srgb_format); bool write_compressed_texture_file(const char* pFilename, const gpu_image_vec& g, bool use_srgb_format); bool write_compressed_texture_file(const char *pFilename, const gpu_image &g, bool use_srgb_format); - + bool write_3dfx_out_file(const char* pFilename, const gpu_image& gi); // GPU texture block unpacking @@ -145,8 +150,15 @@ namespace basisu void unpack_bc4(const void *pBlock_bits, uint8_t *pPixels, uint32_t stride); bool unpack_bc3(const void *pBlock_bits, color_rgba *pPixels); void unpack_bc5(const void *pBlock_bits, color_rgba *pPixels); + +#if 0 bool unpack_bc7_mode6(const void *pBlock_bits, color_rgba *pPixels); + int determine_bc7_mode(const void* pBlock); + int determine_bc7_mode_4_index_mode(const void* pBlock); + int determine_bc7_mode_4_or_5_rotation(const void* pBlock); bool unpack_bc7(const void* pBlock_bits, color_rgba* pPixels); // full format +#endif + bool unpack_bc6h(const void* pSrc_block, void* pDst_block, bool is_signed, uint32_t dest_pitch_in_halfs = 4 * 3); // full format, outputs HALF values, RGB texels only (not RGBA) void unpack_atc(const void* pBlock_bits, color_rgba* pPixels); // We only support CC_MIXED non-alpha blocks here because that's the only mode the transcoder uses at the moment. @@ -155,15 +167,18 @@ namespace basisu bool unpack_pvrtc2(const void* p, color_rgba* pPixels); void unpack_etc2_eac_r(const void *p, color_rgba* pPixels, uint32_t c); void unpack_etc2_eac_rg(const void* p, color_rgba* pPixels); - + // unpack_block() is primarily intended to unpack texture data created by the transcoder. // For some texture formats (like ETC2 RGB, PVRTC2, FXT1) it's not yet a complete implementation. // Unpacks LDR texture formats only. - bool unpack_block(texture_format fmt, const void *pBlock, color_rgba *pPixels); + bool unpack_block(texture_format fmt, const void *pBlock, color_rgba *pPixels, bool astc_srgb); // Unpacks HDR texture formats only. bool unpack_block_hdr(texture_format fmt, const void* pBlock, vec4F* pPixels); - + + bool read_astc_file(const uint8_t* pImage_data, size_t image_data_size, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height); + bool read_astc_file(const char* pFilename, vector2D& blocks, uint32_t& block_width, uint32_t& block_height, uint32_t& width, uint32_t& height); bool write_astc_file(const char* pFilename, const void* pBlocks, uint32_t block_width, uint32_t block_height, uint32_t dim_x, uint32_t dim_y); - + } // namespace basisu + diff --git a/encoder/basisu_math.h b/encoder/basisu_math.h index 66bb749..24b8385 100644 --- a/encoder/basisu_math.h +++ b/encoder/basisu_math.h @@ -8,10 +8,10 @@ namespace bu_math // Would prefer using SSE1 etc. but that would require implementing multiple versions and platform divergence (needing more testing). BASISU_FORCE_INLINE float inv_sqrt(float v) { - union - { - float flt; - uint32_t ui; + union + { + float flt; + uint32_t ui; } un; un.flt = v; @@ -20,6 +20,16 @@ namespace bu_math return 0.703952253f * un.flt * (2.38924456f - v * (un.flt * un.flt)); } + inline float linstep(float edge0, float edge1, float x) + { + assert(edge1 != edge0); + + // Scale, and clamp x to 0..1 range + x = basisu::saturate((x - edge0) / (edge1 - edge0)); + + return x; + } + inline float smoothstep(float edge0, float edge1, float x) { assert(edge1 != edge0); @@ -1130,12 +1140,12 @@ namespace bu_math template Z& matrix_mul_helper(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_rows); - static_assert((int)Z::num_cols == (int)Y::num_cols); - static_assert((int)X::num_cols == (int)Y::num_rows); + static_assert(Z::num_rows == X::num_rows); + static_assert(Z::num_cols == Y::num_cols); + static_assert(X::num_cols == Y::num_rows); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_rows; r++) - for (int c = 0; c < Y::num_cols; c++) + for (uint32_t r = 0; r < X::num_rows; r++) + for (uint32_t c = 0; c < Y::num_cols; c++) { typename Z::scalar_type s = lhs(r, 0) * rhs(0, c); for (uint32_t i = 1; i < X::num_cols; i++) @@ -1148,12 +1158,12 @@ namespace bu_math template Z& matrix_mul_helper_transpose_lhs(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_cols); - static_assert((int)Z::num_cols == (int)Y::num_cols); - static_assert((int)X::num_rows == (int)Y::num_rows); + static_assert(Z::num_rows == X::num_cols); + static_assert(Z::num_cols == Y::num_cols); + static_assert(X::num_rows == Y::num_rows); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_cols; r++) - for (int c = 0; c < Y::num_cols; c++) + for (uint32_t r = 0; r < X::num_cols; r++) + for (uint32_t c = 0; c < Y::num_cols; c++) { typename Z::scalar_type s = lhs(0, r) * rhs(0, c); for (uint32_t i = 1; i < X::num_rows; i++) @@ -1166,12 +1176,12 @@ namespace bu_math template Z& matrix_mul_helper_transpose_rhs(Z& result, const X& lhs, const Y& rhs) { - static_assert((int)Z::num_rows == (int)X::num_rows); - static_assert((int)Z::num_cols == (int)Y::num_rows); - static_assert((int)X::num_cols == (int)Y::num_cols); + static_assert(Z::num_rows == X::num_rows); + static_assert(Z::num_cols == Y::num_rows); + static_assert(X::num_cols == Y::num_cols); assert(((void*)&result != (void*)&lhs) && ((void*)&result != (void*)&rhs)); - for (int r = 0; r < X::num_rows; r++) - for (int c = 0; c < Y::num_rows; c++) + for (uint32_t r = 0; r < X::num_rows; r++) + for (uint32_t c = 0; c < Y::num_rows; c++) { typename Z::scalar_type s = lhs(r, 0) * rhs(c, 0); for (uint32_t i = 1; i < X::num_cols; i++) @@ -1180,17 +1190,21 @@ namespace bu_math } return result; } - + template class matrix { public: typedef T scalar_type; + static const uint32_t num_rows = R; + static const uint32_t num_cols = C; +#if 0 enum { num_rows = R, num_cols = C }; +#endif typedef vec col_vec; typedef vec < (R > 1) ? (R - 1) : 0, T > subcol_vec; @@ -2144,7 +2158,7 @@ namespace bu_math static inline matrix make_tensor_product_matrix(const row_vec& v, const row_vec& w) { matrix ret; - for (int r = 0; r < num_rows; r++) + for (uint32_t r = 0; r < num_rows; r++) ret[r] = row_vec::mul_components(v.broadcast(r), w); return ret; } @@ -2485,6 +2499,31 @@ namespace basisu int64_t m_total2; }; + class tracked_stat_float + { + public: + tracked_stat_float() { clear(); } + + inline void clear() { m_num = 0; m_total = 0; m_total2 = 0; } + + inline void update(float val) { m_num++; m_total += val; m_total2 += val * val; } + + inline tracked_stat_float& operator += (float val) { update(val); return *this; } + + inline uint32_t get_number_of_values() { return m_num; } + inline float get_total() const { return m_total; } + inline float get_total2() const { return m_total2; } + + inline float get_average() const { return m_num ? m_total / (float)m_num : 0.0f; }; + inline float get_std_dev() const { return m_num ? sqrt((float)(m_num * m_total2 - m_total * m_total)) / m_num : 0.0f; } + inline float get_variance() const { float s = get_std_dev(); return s * s; } + + private: + uint32_t m_num; + float m_total; + float m_total2; + }; + class tracked_stat_dbl { public: @@ -2521,14 +2560,14 @@ namespace basisu FloatType m_mad; // mean absolute deviation FloatType m_min, m_max, m_range; // min and max values, and max-min FloatType m_len; // length of values as a vector (Euclidean norm or L2 norm) - FloatType m_coeff_of_var; // coefficient of variation (std_dev/mean), High CV: Indicates greater variability relative to the mean, meaning the data values are more spread out, + FloatType m_coeff_of_var; // coefficient of variation (std_dev/mean), High CV: Indicates greater variability relative to the mean, meaning the data values are more spread out, // Low CV : Indicates less variability relative to the mean, meaning the data values are more consistent. - - FloatType m_skewness; // Skewness = 0: The data is perfectly symmetric around the mean, - // Skewness > 0: The data is positively skewed (right-skewed), + + FloatType m_skewness; // Skewness = 0: The data is perfectly symmetric around the mean, + // Skewness > 0: The data is positively skewed (right-skewed), // Skewness < 0: The data is negatively skewed (left-skewed) // 0-.5 approx. symmetry, .5-1 moderate skew, >= 1 highly skewed - + FloatType m_kurtosis; // Excess Kurtosis: Kurtosis = 0: The distribution has normal kurtosis (mesokurtic) // Kurtosis > 0: The distribution is leptokurtic, with heavy tails and a sharp peak // Kurtosis < 0: The distribution is platykurtic, with light tails and a flatter peak @@ -2538,9 +2577,12 @@ namespace basisu FloatType m_median; uint32_t m_median_index; - stats() - { - clear(); + FloatType m_five_percent_lo; // avg of the lowest 5%, must calc median to be valid + FloatType m_five_percent_hi; // avg of the lowest 5%, must calc median to be valid + + stats() + { + clear(); } void clear() @@ -2557,9 +2599,12 @@ namespace basisu m_skewness = 0; m_kurtosis = 0; m_any_zero = false; - + m_median = 0; m_median_index = 0; + + m_five_percent_lo = 0; + m_five_percent_hi = 0; } template @@ -2588,13 +2633,26 @@ namespace basisu m_median = (m_median + vals[(n / 2) - 1].first) * .5f; m_median_index = vals[n / 2].second; + + // sum and avg low 5% and high 5% + const uint32_t p5_n = clamp((n + 10) / 20, 1u, n); + FloatType lo5_sum = 0, hi5_sum = 0; + + for (uint32_t i = 0; i < p5_n; i++) + { + lo5_sum += vals[i].first; + hi5_sum += vals[n - 1 - i].first; + } + + m_five_percent_lo = lo5_sum / FloatType(p5_n); + m_five_percent_hi = hi5_sum / FloatType(p5_n); } template void calc(uint32_t n, const T* pVals, uint32_t stride = 1, bool calc_median_flag = false) { clear(); - + if (!n) return; @@ -2609,10 +2667,10 @@ namespace basisu if (v == 0.0f) m_any_zero = true; - + m_total += v; m_total_sq += v * v; - + if (!i) { m_min = v; @@ -2634,12 +2692,12 @@ namespace basisu m_avg = m_total / nd; m_avg_sq = m_total_sq / nd; m_rms = sqrt(m_avg_sq); - + for (uint32_t i = 0; i < n; i++) { FloatType v = (FloatType)pVals[i * stride]; FloatType d = v - m_avg; - + const FloatType d2 = d * d; const FloatType d3 = d2 * d; const FloatType d4 = d3 * d; @@ -2680,6 +2738,55 @@ namespace basisu m_total += v; } + + const FloatType nd = (FloatType)n; + + m_avg = m_total / nd; + + for (uint32_t i = 0; i < n; i++) + { + FloatType v = (FloatType)pVals[i * stride]; + FloatType d = v - m_avg; + + const FloatType d2 = d * d; + + m_var += d2; + } + + m_var /= nd; + m_std_dev = sqrt(m_var); + } + + // Only compute average, variance and standard deviation. + template + void calc_simplified_with_range(uint32_t n, const T* pVals, uint32_t stride = 1) + { + clear(); + + if (!n) + return; + + m_n = n; + + for (uint32_t i = 0; i < n; i++) + { + FloatType v = (FloatType)pVals[i * stride]; + + m_total += v; + + if (!i) + { + m_min = v; + m_max = v; + } + else + { + m_min = minimum(m_min, v); + m_max = maximum(m_max, v); + } + } + + m_range = m_max - m_min; const FloatType nd = (FloatType)n; @@ -2712,7 +2819,7 @@ namespace basisu FloatType m_euclidean_dist; // euclidean distance between values as vectors FloatType m_cosine_sim; // normalized dot products of values as vectors FloatType m_min_diff, m_max_diff; // minimum/maximum abs difference between values - + comparative_stats() { clear(); @@ -2738,7 +2845,7 @@ namespace basisu clear(); if (!n) return; - + stats temp_a_stats; if (!pA_stats) { @@ -2757,7 +2864,7 @@ namespace basisu { const FloatType fa = (FloatType)pA[i * a_stride]; const FloatType fb = (FloatType)pB[i * b_stride]; - + if ((pA_stats->m_min >= 0.0f) && (pB_stats->m_min >= 0.0f)) { const FloatType ld = log(fa + 1.0f) - log(fb + 1.0f); @@ -2766,7 +2873,7 @@ namespace basisu const FloatType diff = fa - fb; const FloatType abs_diff = fabs(diff); - + m_mse += diff * diff; m_mae += abs_diff; @@ -2781,7 +2888,7 @@ namespace basisu } const FloatType nd = (FloatType)n; - + m_euclidean_dist = sqrt(m_mse); m_mse /= nd; @@ -2790,7 +2897,7 @@ namespace basisu m_mae /= nd; m_cov /= nd; - + FloatType dv = (pA_stats->m_std_dev * pB_stats->m_std_dev); if (dv != 0.0f) m_pearson = m_cov / dv; @@ -2883,9 +2990,9 @@ namespace basisu const FloatType fb = (FloatType)pB[i * b_stride]; const FloatType diff = fa - fb; - + m_mse += diff * diff; - + const FloatType da = fa - pA_stats->m_avg; const FloatType db = fb - pB_stats->m_avg; m_cov += da * db; @@ -2897,7 +3004,7 @@ namespace basisu m_mse /= nd; m_rmse = sqrt(m_mse); - + m_cov /= nd; } @@ -2938,7 +3045,7 @@ namespace basisu m_cov /= nd; } }; - + class stat_history { public: @@ -3083,12 +3190,12 @@ namespace basisu uint32_t lowerBits = float_union.u & 0xFFFF; // Round to nearest or even - if ((lowerBits & 0x8000) && + if ((lowerBits & 0x8000) && ((lowerBits > 0x8000) || ((lowerBits == 0x8000) && (upperBits & 1))) ) { // Round up - upperBits += 1; + upperBits += 1; // Check for overflow in the exponent after rounding up if (((upperBits & 0x7F80) == 0x7F80) && ((upperBits & 0x007F) == 0)) @@ -3140,6 +3247,7 @@ namespace basisu return res; } - - + + } // namespace basisu + diff --git a/encoder/basisu_uastc_enc.cpp b/encoder/basisu_uastc_enc.cpp index 701534c..88448ee 100644 --- a/encoder/basisu_uastc_enc.cpp +++ b/encoder/basisu_uastc_enc.cpp @@ -1,5 +1,5 @@ // basisu_uastc_enc.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -223,7 +223,7 @@ namespace basisu default: break; } -#endif +#endif uint32_t total_planes = 1; switch (result.m_uastc_mode) @@ -319,7 +319,15 @@ namespace basisu const uint32_t comp_plane = (total_comps == 2) ? c : ((c == result.m_astc.m_ccs) ? 1 : 0); if (comp_plane == plane_index) - std::swap(endpoints[c * 2 + 0], endpoints[c * 2 + 1]); + { + // shut up a useless gcc warning + assert((c * 2 + 1) < (int)sizeof(endpoints)); + + if ((c * 2 + 1) < (int)sizeof(endpoints)) + { + std::swap(endpoints[c * 2 + 0], endpoints[c * 2 + 1]); + } + } } } else @@ -456,7 +464,7 @@ namespace basisu printf("Total bits: %u, endpoint bits: %u, weight bits: %u\n", block_bit_offset, total_endpoint_bits, total_weight_bits); #endif } - + // MODE 0 // 0. DualPlane: 0, WeightRange: 8 (16), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 19 (192) MODE6 RGB // 18. DualPlane: 0, WeightRange: 11 (32), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 11 (32) MODE6 RGB @@ -507,7 +515,7 @@ namespace basisu astc_results.m_endpoints[3] = ccell_results.m_astc_high_endpoint.m_c[1]; astc_results.m_endpoints[4] = ccell_results.m_astc_low_endpoint.m_c[2]; astc_results.m_endpoints[5] = ccell_results.m_astc_high_endpoint.m_c[2]; - + bool invert = false; if (pForce_selectors == nullptr) @@ -1128,7 +1136,7 @@ namespace basisu } // common_pattern } - // MODE 5 + // MODE 5 // DualPlane: 0, WeightRange: 5 (8), Subsets: 1, CEM: 8 (RGB Direct ), EndpointRange: 20 (256) BC7 MODE 6 (or MODE 1 1-subset) static void astc_mode5(const color_rgba block[4][4], uastc_encode_results* pResults, uint32_t& total_results, bc7enc_compress_block_params& comp_params) { @@ -1259,7 +1267,7 @@ namespace basisu ccell_results_rgb.m_pSelectors_temp = &ccell_result_selectors_temp[0]; uint64_t part_err_rgb = color_cell_compression(255, &ccell_params_rgb, &ccell_results_rgb, &comp_params); - + color_cell_compressor_params ccell_params_a; memset(&ccell_params_a, 0, sizeof(ccell_params_a)); @@ -1416,9 +1424,9 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { const uint32_t astc_part = bc7_convert_partition_index_3_to_2(g_bc7_partition3[16 * bc7_pattern + x + y * 4], common_pattern_k); -#ifdef _DEBUG +#ifdef _DEBUG assert((int)astc_part == astc_compute_texel_partition(astc_pattern, x, y, 0, 2, true)); -#endif +#endif part_pixel_index[y][x] = num_part_pixels[astc_part]; part_pixels[astc_part][num_part_pixels[astc_part]++] = block[y][x]; @@ -1583,7 +1591,7 @@ namespace basisu } #endif } - + // 9. DualPlane: 0, WeightRange: 2 (4), Subsets: 2, CEM: 12 (RGBA Direct), EndpointRange: 8 (16) - BC7 MODE 7 // 16. DualPlane: 0, WeightRange : 2 (4), Subsets : 2, CEM: 4 (LA Direct), EndpointRange : 20 (256) - BC7 MODE 7 static void astc_mode9_or_16(uint32_t mode, const color_rgba source_block[4][4], uastc_encode_results* pResults, uint32_t& total_results, bc7enc_compress_block_params& comp_params, uint32_t estimate_partition_list_size) @@ -2499,7 +2507,7 @@ namespace basisu total_results++; } } - + static void compute_block_error(const color_rgba block[4][4], const color_rgba decoded_block[4][4], uint64_t &total_rgb_err, uint64_t &total_rgba_err, uint64_t &total_la_err) { uint64_t total_err_r = 0, total_err_g = 0, total_err_b = 0, total_err_a = 0; @@ -2542,18 +2550,18 @@ namespace basisu color_rgba tblock_bc1[4][4]; dxt1_block tbc1_block[8]; basist::encode_bc1(tbc1_block, (const uint8_t*)&decoded_uastc_block[0][0], 0); - unpack_block(texture_format::cBC1, tbc1_block, &tblock_bc1[0][0]); + unpack_block(texture_format::cBC1, tbc1_block, &tblock_bc1[0][0], false); color_rgba tblock_hint0_bc1[4][4]; color_rgba tblock_hint1_bc1[4][4]; - + etc_block etc1_blk; memset(&etc1_blk, 0, sizeof(etc1_blk)); eac_a8_block etc2_blk; memset(&etc2_blk, 0, sizeof(etc2_blk)); etc2_blk.m_multiplier = 1; - + // Pack to UASTC, then unpack, because the endpoints may be swapped. uastc_block temp_ublock; @@ -2561,7 +2569,7 @@ namespace basisu unpacked_uastc_block temp_ublock_unpacked; unpack_uastc(temp_ublock, temp_ublock_unpacked, false); - + unpacked_uastc_block ublock; memset(&ublock, 0, sizeof(ublock)); ublock.m_mode = best_results.m_uastc_mode; @@ -2579,7 +2587,7 @@ namespace basisu { transcode_uastc_to_bc1_hint1(ublock, (color32 (*)[4]) decoded_uastc_block, &b, false); - unpack_block(texture_format::cBC1, &b, &tblock_hint1_bc1[0][0]); + unpack_block(texture_format::cBC1, &b, &tblock_hint1_bc1[0][0], false); } // HINT0 @@ -2590,8 +2598,8 @@ namespace basisu else { transcode_uastc_to_bc1_hint0(ublock, &b); - - unpack_block(texture_format::cBC1, &b, &tblock_hint0_bc1[0][0]); + + unpack_block(texture_format::cBC1, &b, &tblock_hint0_bc1[0][0], false); } // Compute block errors @@ -2612,7 +2620,7 @@ namespace basisu const float err_thresh0 = 1.075f; const float err_thresh1 = 1.075f; - + if ((g_uastc_mode_has_bc1_hint0[best_mode]) && (t_err_hint0 <= t_err * err_thresh0)) bc1_hint0 = true; @@ -2779,7 +2787,7 @@ namespace basisu uint32_t first_flip = 0, last_flip = 2; uint32_t first_individ = 0, last_individ = 2; - + if (flags & cPackUASTCETC1DisableFlipAndIndividual) { last_flip = 1; @@ -2791,7 +2799,7 @@ namespace basisu first_flip = 1; last_flip = first_flip + 1; } - + for (uint32_t flip = first_flip; flip < last_flip; flip++) { trial_block.set_flip_bit(flip != 0); @@ -2799,7 +2807,7 @@ namespace basisu for (uint32_t individ = first_individ; individ < last_individ; individ++) { const uint32_t mul = individ ? 15 : 31; - + trial_block.set_diff_bit(individ == 0); color_rgba unbiased_block_colors[2]; @@ -2815,7 +2823,7 @@ namespace basisu { const etc_coord2 &c = g_etc1_pixel_coords[flip][subset][j]; const color_rgba& p = decoded_uastc_block[c.m_y][c.m_x]; - + avg_color[0] += p.r; avg_color[1] += p.g; avg_color[2] += p.b; @@ -2833,13 +2841,13 @@ namespace basisu unbiased_block_colors[subset][1] = (uint8_t)((avg_color[1] * mul + 1020) / (8 * 255)); unbiased_block_colors[subset][2] = (uint8_t)((avg_color[2] * mul + 1020) / (8 * 255)); unbiased_block_colors[subset][3] = 0; - + } // subset - + for (uint32_t bias_iter = 0; bias_iter < last_bias; bias_iter++) { const uint32_t bias = use_faster_bias_mode_table ? s_sorted_bias_modes[bias_iter] : bias_iter; - + color_rgba block_colors[2]; for (uint32_t subset = 0; subset < 2; subset++) block_colors[subset] = has_bias ? apply_etc1_bias((color32&)unbiased_block_colors[subset], bias, mul, subset) : unbiased_block_colors[subset]; @@ -2873,7 +2881,7 @@ namespace basisu uint64_t best_subset_err = UINT64_MAX; const uint32_t inten_table_limit = (level == cPackUASTCLevelVerySlow) ? 8 : ((range[subset] > 51) ? 8 : (range[subset] >= 7 ? 4 : 2)); - + for (uint32_t inten_table = 0; inten_table < inten_table_limit; inten_table++) { trial_block.set_inten_table(subset, inten_table); @@ -3008,7 +3016,7 @@ namespace basisu uint32_t m_table; uint32_t m_multiplier; }; - + static uint64_t uastc_pack_eac_a8(uastc_pack_eac_a8_results& results, const uint8_t* pPixels, uint32_t num_pixels, uint32_t base_search_rad, uint32_t mul_search_rad, uint32_t table_mask) { assert(num_pixels <= 16); @@ -3152,7 +3160,7 @@ namespace basisu solid_results.m_common_pattern = 0; solid_results.m_solid_color = first_color; memset(&solid_results.m_astc, 0, sizeof(solid_results.m_astc)); - + etc_block etc1_blk; uint32_t etc1_bias = 0; @@ -3168,17 +3176,17 @@ namespace basisu return; } - + int level = flags & 7; const bool favor_uastc_error = (flags & cPackUASTCFavorUASTCError) != 0; const bool favor_bc7_error = !favor_uastc_error && ((flags & cPackUASTCFavorBC7Error) != 0); //const bool etc1_perceptual = true; - + // TODO: This uses 64KB of stack space! uastc_encode_results results[MAX_ENCODE_RESULTS]; - + level = clampi(level, cPackUASTCLevelFastest, cPackUASTCLevelVerySlow); - + // Set all options to slowest, then configure from there depending on the selected level. uint32_t mode_mask = UINT32_MAX; uint32_t uber_level = 6; @@ -3189,12 +3197,12 @@ namespace basisu uint32_t least_squares_passes = 2; bool bc1_hints = true; bool only_use_la_on_transparent_blocks = false; - + switch (level) { case cPackUASTCLevelFastest: { - mode_mask = (1 << 0) | (1 << 8) | + mode_mask = (1 << 0) | (1 << 8) | (1 << 11) | (1 << 12) | (1 << 15); always_try_alpha_modes = false; @@ -3220,7 +3228,7 @@ namespace basisu estimate_partition = true; break; } - case cPackUASTCLevelDefault: + case cPackUASTCLevelDefault: { mode_mask = (1 << 0) | (1 << 1) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13) | @@ -3258,9 +3266,9 @@ namespace basisu // HACK HACK //mode_mask &= ~(1 << 18); //mode_mask = (1 << 18)| (1 << 10); - + uint32_t total_results = 0; - + if (only_use_la_on_transparent_blocks) { if ((is_la) && (!has_alpha)) @@ -3268,7 +3276,7 @@ namespace basisu } const bool try_alpha_modes = has_alpha || always_try_alpha_modes; - + bc7enc_compress_block_params comp_params; memset(&comp_params, 0, sizeof(comp_params)); comp_params.m_max_partitions_mode1 = 64; @@ -3343,7 +3351,7 @@ namespace basisu } assert(total_results); - + // Fix up the errors so we consistently have LA, RGB, or RGBA error. for (uint32_t i = 0; i < total_results; i++) { @@ -3377,7 +3385,7 @@ namespace basisu } } } - + unpacked_uastc_block unpacked_ublock; memset(&unpacked_ublock, 0, sizeof(unpacked_ublock)); @@ -3447,7 +3455,7 @@ namespace basisu encode_bc7_block(&bc7_data, &bc7_results); color_rgba decoded_bc7_block[4][4]; - unpack_block(texture_format::cBC7, &bc7_data, &decoded_bc7_block[0][0]); + unpack_block(texture_format::cBC7, &bc7_data, &decoded_bc7_block[0][0], false); // Compute BC7 error uint64_t total_bc7_la_err, total_bc7_rgb_err, total_bc7_rgba_err; @@ -3544,7 +3552,7 @@ namespace basisu const uastc_encode_results& best_results = results[best_index]; const uint32_t best_mode = best_results.m_uastc_mode; const astc_block_desc& best_astc_results = best_results.m_astc; - + color_rgba decoded_uastc_block[4][4]; bool success = unpack_uastc(best_mode, best_results.m_common_pattern, best_results.m_solid_color.get_color32(), best_astc_results, (basist::color32 *)&decoded_uastc_block[0][0], false); (void)success; @@ -3562,14 +3570,14 @@ namespace basisu basist::uastc_block temp_block; pack_uastc(temp_block, best_results, etc1_blk, 0, etc_eac_a8_blk, false, false); - + basist::color32 temp_block_unpacked[4][4]; success = basist::unpack_uastc(temp_block, (basist::color32 *)temp_block_unpacked, false); VALIDATE(success); // Now round trip to packed ASTC and back, then decode to pixels. uint32_t astc_data[4]; - + if (best_results.m_uastc_mode == UASTC_MODE_INDEX_SOLID_COLOR) pack_astc_solid_block(astc_data, (color32 &)best_results.m_solid_color); else @@ -3587,7 +3595,7 @@ namespace basisu for (uint32_t x = 0; x < 4; x++) { VALIDATE(decoded_astc_block[y][x] == decoded_uastc_block[y][x]); - + VALIDATE(temp_block_unpacked[y][x].c[0] == decoded_uastc_block[y][x].r); VALIDATE(temp_block_unpacked[y][x].c[1] == decoded_uastc_block[y][x].g); VALIDATE(temp_block_unpacked[y][x].c[2] == decoded_uastc_block[y][x].b); @@ -3601,7 +3609,7 @@ namespace basisu bool bc1_hint0 = false, bc1_hint1 = false; if (bc1_hints) compute_bc1_hints(bc1_hint0, bc1_hint1, best_results, block, decoded_uastc_block); - + eac_a8_block eac_a8_blk; if ((g_uastc_mode_has_alpha[best_mode]) && (best_mode != UASTC_MODE_INDEX_SOLID_COLOR)) { @@ -3613,7 +3621,7 @@ namespace basisu uastc_pack_eac_a8_results eac8_a8_results; memset(&eac8_a8_results, 0, sizeof(eac8_a8_results)); uastc_pack_eac_a8(eac8_a8_results, decoded_uastc_block_alpha, 16, 0, eac_a8_mul_search_rad, eac_a8_table_mask); - + // All we care about for hinting is the table and multiplier. eac_a8_blk.m_table = eac8_a8_results.m_table; eac_a8_blk.m_multiplier = eac8_a8_results.m_multiplier; @@ -3810,11 +3818,11 @@ namespace basisu { std::size_t operator()(selector_bitsequence const& s) const noexcept { - return hash_hsieh((const uint8_t*)&s, sizeof(s)); + return basist::hash_hsieh((const uint8_t*)&s, sizeof(s)); } }; - - static bool uastc_rdo_blocks(uint32_t first_index, uint32_t last_index, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, + + static bool uastc_rdo_blocks(uint32_t first_index, uint32_t last_index, basist::uastc_block* pBlocks, const color_rgba* pBlock_pixels, const uastc_rdo_params& params, uint32_t flags, uint32_t &total_skipped, uint32_t &total_refined, uint32_t &total_modified, uint32_t &total_smooth) { debug_printf("uastc_rdo_blocks: Processing blocks %u to %u\n", first_index, last_index); @@ -3823,7 +3831,7 @@ namespace basisu const bool perceptual = false; std::unordered_map selector_history; - + for (uint32_t block_index = first_index; block_index < last_index; block_index++) { const basist::uastc_block& blk = pBlocks[block_index]; @@ -3872,8 +3880,8 @@ namespace basisu basist::encode_bc7_block(&b7_block, &b7_results); color_rgba decoded_b7_blk[4][4]; - unpack_block(texture_format::cBC7, &b7_block, &decoded_b7_blk[0][0]); - + unpack_block(texture_format::cBC7, &b7_block, &decoded_b7_blk[0][0], false); + uint64_t bc7_err = 0; for (uint32_t i = 0; i < 16; i++) bc7_err += color_distance(perceptual, pPixels[i], ((color_rgba*)decoded_b7_blk)[i], true); @@ -3928,7 +3936,7 @@ namespace basisu float best_t = cur_ms_err * smooth_block_error_scale + cur_bits * params.m_lambda; - // Now scan through previous blocks, insert their selector bit patterns into the current block, and find + // Now scan through previous blocks, insert their selector bit patterns into the current block, and find // selector bit patterns which don't increase the overall block error too much. for (int prev_block_index = last_block_to_check; prev_block_index >= first_block_to_check; --prev_block_index) { @@ -3981,7 +3989,7 @@ namespace basisu basist::encode_bc7_block(&trial_b7_block, &trial_b7_results); color_rgba decoded_trial_b7_blk[4][4]; - unpack_block(texture_format::cBC7, &trial_b7_block, &decoded_trial_b7_blk[0][0]); + unpack_block(texture_format::cBC7, &trial_b7_block, &decoded_trial_b7_blk[0][0], false); uint64_t trial_bc7_err = 0; for (uint32_t i = 0; i < 16; i++) @@ -4050,7 +4058,7 @@ namespace basisu color_rgba decoded_trial_uastc_block[4][4]; bool success = unpack_uastc(results.m_uastc_mode, results.m_common_pattern, results.m_solid_color.get_color32(), results.m_astc, (basist::color32*) & decoded_trial_uastc_block[0][0], false); assert(success); - + BASISU_NOTE_UNUSED(success); uint64_t trial_uastc_err = 0; @@ -4077,7 +4085,7 @@ namespace basisu // Write the modified block pBlocks[block_index] = best_block; - + } // if (best_block_index != block_index) { @@ -4093,8 +4101,8 @@ namespace basisu return true; } - - // This function implements a basic form of rate distortion optimization (RDO) for UASTC. + + // This function implements a basic form of rate distortion optimization (RDO) for UASTC. // It only changes selectors and then updates the hints. It uses very approximate LZ bitprice estimation. // There's A LOT that can be done better in here, but it's a start. // One nice advantage of the method used here is that it works for any input, no matter which or how many modes it uses. @@ -4133,7 +4141,7 @@ namespace basisu { std::lock_guard lck(stat_mutex); - + all_succeeded = all_succeeded && status; total_skipped += job_skipped; total_modified += job_modified; @@ -4152,7 +4160,12 @@ namespace basisu } debug_printf("uastc_rdo: Total modified: %3.2f%%, total skipped: %3.2f%%, total refined: %3.2f%%, total smooth: %3.2f%%\n", total_modified * 100.0f / num_blocks, total_skipped * 100.0f / num_blocks, total_refined * 100.0f / num_blocks, total_smooth * 100.0f / num_blocks); - + return status; } } // namespace basisu + + + + + diff --git a/encoder/cppspmd_flow.h b/encoder/cppspmd_flow.h index 07b5924..3e83e9e 100644 --- a/encoder/cppspmd_flow.h +++ b/encoder/cppspmd_flow.h @@ -48,7 +48,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_return() m_kernel_exec = andnot(m_exec, m_kernel_exec); m_exec = exec_mask::all_off(); } - + template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_unmasked(const UnmaskedBody& unmaskedBody) { @@ -61,7 +61,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_unmasked(const UnmaskedBody& unmaske m_kernel_exec = m_kernel_exec & orig_kernel_exec; m_exec = m_exec & orig_exec; - + check_masks(); } @@ -69,9 +69,9 @@ struct scoped_unmasked_restorer { spmd_kernel *m_pKernel; exec_mask m_orig_exec, m_orig_kernel_exec; - - CPPSPMD_FORCE_INLINE scoped_unmasked_restorer(spmd_kernel *pKernel) : - m_pKernel(pKernel), + + CPPSPMD_FORCE_INLINE scoped_unmasked_restorer(spmd_kernel *pKernel) : + m_pKernel(pKernel), m_orig_exec(pKernel->m_exec), m_orig_kernel_exec(pKernel->m_kernel_exec) { @@ -79,15 +79,15 @@ struct scoped_unmasked_restorer pKernel->m_exec = exec_mask::all_on(); } - CPPSPMD_FORCE_INLINE ~scoped_unmasked_restorer() - { + CPPSPMD_FORCE_INLINE ~scoped_unmasked_restorer() + { m_pKernel->m_kernel_exec = m_pKernel->m_kernel_exec & m_orig_kernel_exec; m_pKernel->m_exec = m_pKernel->m_exec & m_orig_exec; m_pKernel->check_masks(); } }; -#define SPMD_UNMASKED_BEGIN { scoped_unmasked_restorer _unmasked_restorer(this); +#define SPMD_UNMASKED_BEGIN { scoped_unmasked_restorer _unmasked_restorer(this); #define SPMD_UNMASKED_END } #if 0 @@ -113,9 +113,9 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_if_break(const vbool& cond) #ifdef _DEBUG assert(m_in_loop); #endif - + exec_mask cond_exec(cond); - + m_exec = andnot(m_exec & cond_exec, m_exec); check_masks(); @@ -157,7 +157,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_sifelse(const vbool& cond, const IfB m_exec = em; elseBody(); } - + m_exec = orig_exec; } @@ -165,7 +165,7 @@ template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_if(const vbool& cond, const IfBody& ifBody) { exec_mask cond_exec(cond); - + exec_mask pre_if_exec = cond_exec & m_exec; if (any(pre_if_exec)) @@ -188,7 +188,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_ifelse(const vbool& cond, const IfBo bool all_flag = false; exec_mask cond_exec(cond); - + { exec_mask pre_if_exec = cond_exec & m_exec; @@ -218,9 +218,10 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_ifelse(const vbool& cond, const IfBo exec_mask unexecuted_lanes = cond_exec & m_exec; m_exec = pre_if_exec; - ifBody(); + // 11/22/2025: changed to elseBody() here, simple bug, we use the macro variants of ifelse anyway + elseBody(); - // Propagate any lanes that got disabled inside the if body into the exec mask outside the if body, but turn on any lanes that didn't execute inside the if body. + // Propagate any lanes that got disabled inside the else body into the exec mask outside the else body, but turn on any lanes that didn't execute inside the else body. m_exec = m_exec | unexecuted_lanes; check_masks(); @@ -290,17 +291,17 @@ struct scoped_exec_restorer2 { spmd_kernel *m_pKernel; exec_mask m_unexecuted_lanes; - - CPPSPMD_FORCE_INLINE scoped_exec_restorer2(spmd_kernel *pKernel, const vbool &cond) : + + CPPSPMD_FORCE_INLINE scoped_exec_restorer2(spmd_kernel *pKernel, const vbool &cond) : m_pKernel(pKernel) - { + { exec_mask cond_exec(cond); m_unexecuted_lanes = andnot(cond_exec, pKernel->m_exec); pKernel->m_exec = cond_exec & pKernel->m_exec; } - CPPSPMD_FORCE_INLINE ~scoped_exec_restorer2() - { + CPPSPMD_FORCE_INLINE ~scoped_exec_restorer2() + { m_pKernel->m_exec = m_pKernel->m_exec | m_unexecuted_lanes; m_pKernel->check_masks(); } @@ -327,17 +328,17 @@ public: inline scoped_exec_saver(spmd_kernel *pKernel) : m_exec(pKernel->m_exec), m_kernel_exec(pKernel->m_kernel_exec), m_continue_mask(pKernel->m_continue_mask), m_pKernel(pKernel) - { + { #ifdef _DEBUG m_in_loop = pKernel->m_in_loop; #endif } - + inline ~scoped_exec_saver() - { - m_pKernel->m_exec = m_exec; - m_pKernel->m_continue_mask = m_continue_mask; - m_pKernel->m_kernel_exec = m_kernel_exec; + { + m_pKernel->m_exec = m_exec; + m_pKernel->m_continue_mask = m_continue_mask; + m_pKernel->m_kernel_exec = m_kernel_exec; #ifdef _DEBUG m_pKernel->m_in_loop = m_in_loop; m_pKernel->check_masks(); @@ -353,7 +354,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo { if (begin == end) return; - + if (!any(m_exec)) return; @@ -362,12 +363,12 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo std::swap(begin, end); exec_mask prev_continue_mask = m_continue_mask, prev_exec = m_exec; - + int total_full = (end - begin) / PROGRAM_COUNT; int total_partial = (end - begin) % PROGRAM_COUNT; lint_t loop_index = begin + program_index; - + const int total_loops = total_full + (total_partial ? 1 : 0); m_continue_mask = exec_mask::all_off(); @@ -390,7 +391,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_foreach(int begin, int end, const Fo m_continue_mask = exec_mask::all_off(); check_masks(); - + store_all(loop_index, loop_index + PROGRAM_COUNT); } @@ -443,9 +444,9 @@ struct scoped_while_restorer #ifdef _DEBUG bool m_prev_in_loop; #endif - - CPPSPMD_FORCE_INLINE scoped_while_restorer(spmd_kernel *pKernel) : - m_pKernel(pKernel), + + CPPSPMD_FORCE_INLINE scoped_while_restorer(spmd_kernel *pKernel) : + m_pKernel(pKernel), m_orig_exec(pKernel->m_exec), m_orig_continue_mask(pKernel->m_continue_mask) { @@ -457,8 +458,8 @@ struct scoped_while_restorer #endif } - CPPSPMD_FORCE_INLINE ~scoped_while_restorer() - { + CPPSPMD_FORCE_INLINE ~scoped_while_restorer() + { m_pKernel->m_exec = m_orig_exec & m_pKernel->m_kernel_exec; m_pKernel->m_continue_mask = m_orig_continue_mask; #ifdef _DEBUG @@ -514,7 +515,7 @@ struct scoped_simple_while_restorer m_pKernel(pKernel), m_orig_exec(pKernel->m_exec) { - + #ifdef _DEBUG m_prev_in_loop = pKernel->m_in_loop; pKernel->m_in_loop = true; @@ -536,18 +537,18 @@ struct scoped_simple_while_restorer #define SPMD_SWHILE(cond) { scoped_simple_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); \ while(true) { \ exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; -#define SPMD_SWEND } } +#define SPMD_SWEND } } // Cannot use SPMD break, continue, or return inside simple do #define SPMD_SDO { scoped_simple_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); while(true) { -#define SPMD_SEND_DO(cond) exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; } } +#define SPMD_SEND_DO(cond) exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(cond)); m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; } } #undef SPMD_FOR #undef SPMD_END_FOR #define SPMD_FOR(for_init, for_cond) { for_init; scoped_while_restorer CPPSPMD_GLUER2(_while_restore_, __LINE__)(this); while(true) { exec_mask CPPSPMD_GLUER2(cond_exec, __LINE__) = exec_mask(vbool(for_cond)); \ m_exec = m_exec & CPPSPMD_GLUER2(cond_exec, __LINE__); if (!any(m_exec)) break; #define SPMD_END_FOR(for_inc) m_exec = m_exec | m_continue_mask; m_continue_mask = exec_mask::all_off(); check_masks(); for_inc; } } - + template CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_for(const ForInitBody& forInitBody, const ForCondBody& forCondBody, const ForIncrBody& forIncrBody, const ForBody& forBody) { @@ -576,7 +577,7 @@ CPPSPMD_FORCE_INLINE void spmd_kernel::spmd_for(const ForInitBody& forInitBody, m_exec = m_exec | m_continue_mask; m_continue_mask = exec_mask::all_off(); check_masks(); - + forIncrBody(); } diff --git a/encoder/jpgd.cpp b/encoder/jpgd.cpp index 57c7ec7..9a534b3 100644 --- a/encoder/jpgd.cpp +++ b/encoder/jpgd.cpp @@ -3,10 +3,11 @@ // Supports box and linear chroma upsampling. // // Released under two licenses. You are free to choose which license you want: -// License 1: +// License 1: // Public Domain // // License 2: +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,6 +25,10 @@ // v2.00, March 20, 2020: Fuzzed with zzuf and afl. Fixed several issues, converted most assert()'s to run-time checks. Added chroma upsampling. Removed freq. domain upsampling. gcc/clang warnings. // +#if defined(__wasi__) +#pragma message("__wasi__ defined in jpgd.cpp: note if a decode error occurs, the app will exit because wasi doesn't support longjmp yet.") +#endif + #include "jpgd.h" #include #include @@ -138,7 +143,7 @@ namespace jpgd { { static void idct(int* pTemp, const jpgd_block_t* pSrc) { - (void)pTemp; + (void)pTemp; (void)pSrc; } }; @@ -253,10 +258,10 @@ namespace jpgd { 8,8,8,8,8,7,6,4, 8,8,8,8,8,7,6,5, 8,8,8,8,8,7,6,6, 8,8,8,8,8,7,7,6, 8,8,8,8,8,8,7,6, 8,8,8,8,8,8,8,6, 8,8,8,8,8,8,8,7, 8,8,8,8,8,8,8,8, }; - static const uint8 s_idct_col_table[] = - { - 1, 1, 2, 3, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 + static const uint8 s_idct_col_table[] = + { + 1, 1, 2, 3, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }; // Scalar "fast pathing" IDCT. @@ -608,7 +613,14 @@ namespace jpgd { { m_error_code = status; free_all_blocks(); + +#ifdef __wasi__ + // HACK HACK for wasi's lack of longjmp support + fprintf(stderr, "jpeg_decoder::stop_decoding: JPEG decode failed with status: %i\n", (int)status); + exit(EXIT_FAILURE); +#else longjmp(m_jmp_state, status); +#endif } void* jpeg_decoder::alloc(size_t nSize, bool zero) @@ -2071,8 +2083,10 @@ namespace jpgd { int jpeg_decoder::decode_next_mcu_row() { +#ifndef __wasi__ if (setjmp(m_jmp_state)) return JPGD_FAILED; +#endif const bool chroma_y_filtering = (m_flags & cFlagLinearChromaFiltering) && ((m_scan_type == JPGD_YH2V2) || (m_scan_type == JPGD_YH1V2)) && (m_image_x_size >= 2) && (m_image_y_size >= 2); if (chroma_y_filtering) @@ -2987,8 +3001,10 @@ namespace jpgd { jpeg_decoder::jpeg_decoder(jpeg_decoder_stream* pStream, uint32_t flags) { +#ifndef __wasi__ if (setjmp(m_jmp_state)) return; +#endif decode_init(pStream, flags); } @@ -3000,8 +3016,10 @@ namespace jpgd { if (m_error_code) return JPGD_FAILED; +#ifndef __wasi__ if (setjmp(m_jmp_state)) return JPGD_FAILED; +#endif decode_start(); diff --git a/encoder/jpgd.h b/encoder/jpgd.h index 92e5333..bd1ad80 100644 --- a/encoder/jpgd.h +++ b/encoder/jpgd.h @@ -1,16 +1,18 @@ // jpgd.h - C++ class for JPEG decompression. -// Public domain, Rich Geldreich +// Dual licensed: Public domain, Rich Geldreich , or Apache 2.0 (see jpgd.cpp) #ifndef JPEG_DECODER_H #define JPEG_DECODER_H #include #include +#ifndef __wasi__ #include +#endif #include #include #ifdef _MSC_VER -#define JPGD_NORETURN __declspec(noreturn) +#define JPGD_NORETURN __declspec(noreturn) #elif defined(__GNUC__) #define JPGD_NORETURN __attribute__ ((noreturn)) #else @@ -140,7 +142,7 @@ namespace jpgd int begin_decoding(); // Returns the next scan line. - // For grayscale images, pScan_line will point to a buffer containing 8-bit pixels (get_bytes_per_pixel() will return 1). + // For grayscale images, pScan_line will point to a buffer containing 8-bit pixels (get_bytes_per_pixel() will return 1). // Otherwise, it will always point to a buffer containing 32-bit RGBA pixels (A will always be 255, and get_bytes_per_pixel() will return 4). // Returns JPGD_SUCCESS if a scan line has been returned. // Returns JPGD_DONE if all scan lines have been returned. @@ -191,7 +193,10 @@ namespace jpgd char m_data[1]; }; + // TODO: we can get rid of longjmp entirely +#ifndef __wasi__ jmp_buf m_jmp_state; +#endif uint32_t m_flags; mem_block* m_pMem_blocks; int m_image_x_size; diff --git a/encoder_lib/encoder_lib.vcxproj b/encoder_lib/encoder_lib.vcxproj index 3ffa09a..90dadde 100644 --- a/encoder_lib/encoder_lib.vcxproj +++ b/encoder_lib/encoder_lib.vcxproj @@ -31,6 +31,8 @@ + + @@ -59,6 +61,8 @@ + + @@ -92,12 +96,14 @@ + + @@ -119,40 +125,40 @@ StaticLibrary true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 StaticLibrary true Unicode - v143 + v145 StaticLibrary true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 StaticLibrary false true Unicode - v143 + v145 @@ -188,7 +194,7 @@ pch.h ..\OpenCL stdcpp17 - StreamingSIMDExtensions2 + AdvancedVectorExtensions @@ -207,7 +213,7 @@ NotUsing pch.h ..\OpenCL - StreamingSIMDExtensions2 + AdvancedVectorExtensions false false stdcpp17 @@ -227,7 +233,7 @@ true NotUsing pch.h - StreamingSIMDExtensions2 + AdvancedVectorExtensions ..\OpenCL Level4 stdcpp17 @@ -267,7 +273,7 @@ pch.h false false - StreamingSIMDExtensions2 + AdvancedVectorExtensions ..\OpenCL Full AnySuitable @@ -313,4 +319,4 @@ - + \ No newline at end of file diff --git a/encoder_lib/encoder_lib.vcxproj.filters b/encoder_lib/encoder_lib.vcxproj.filters index aaee9d9..093809c 100644 --- a/encoder_lib/encoder_lib.vcxproj.filters +++ b/encoder_lib/encoder_lib.vcxproj.filters @@ -1,4 +1,4 @@ - + @@ -99,6 +99,12 @@ Source Files\encoder + + Source Files\encoder + + + Source Files\encoder + @@ -227,6 +233,15 @@ Source Files\encoder + + Source Files\encoder + + + Source Files\encoder + + + Source Files\transcoder + @@ -256,5 +271,8 @@ Source Files\encoder + + Source Files\transcoder + - + \ No newline at end of file diff --git a/example/example.cpp b/example/example.cpp index 5062d3d..d009016 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -1,15 +1,18 @@ // File: example.cpp // This minimal LDR/HDR encoding/transcoder example relies on encoder_lib. It shows how to use the encoder in a few different ways, and the transcoder. -// -// It should be compiled with the preprocessor macros BASISU_SUPPORT_SSE (typically 1) and BASISU_SUPPORT_OPENCL (typically 1). +// +// It should be compiled with the preprocessor macros BASISU_SUPPORT_SSE (typically 1) and BASISU_SUPPORT_OPENCL (typically 1). // They should be set to the same preprocesor options as the encoder. // If OpenCL is enabled, the "..\OpenCL" directory should be in your compiler's include path. Additionally, link against "..\OpenCL\lib\opencl64.lib". #include "../encoder/basisu_comp.h" #include "../transcoder/basisu_transcoder.h" #include "../encoder/basisu_gpu_texture.h" +#include "../encoder/basisu_astc_ldr_encode.h" #define USE_ENCODER (1) +//#define FORCE_SAN_FAILURE + const bool USE_OPENCL = false; // The encoder lives in the "basisu" namespace. @@ -17,7 +20,7 @@ const bool USE_OPENCL = false; using namespace basisu; // Quick function to create a visualization of the Mandelbrot set as an float HDR image. -static void create_mandelbrot(imagef& img) +static void create_mandelbrot(imagef& img) { const int width = 256; const int height = 256; @@ -25,30 +28,30 @@ static void create_mandelbrot(imagef& img) // Create a more interesting color palette uint8_t palette[256][3]; - for (int i = 0; i < 256; i++) + for (int i = 0; i < 256; i++) { - if (i < 64) + if (i < 64) { // Blue to cyan transition palette[i][0] = static_cast(0); // Red component palette[i][1] = static_cast(i * 4); // Green component palette[i][2] = static_cast(255); // Blue component } - else if (i < 128) + else if (i < 128) { // Cyan to green transition palette[i][0] = static_cast(0); // Red component palette[i][1] = static_cast(255); // Green component palette[i][2] = static_cast(255 - (i - 64) * 4); // Blue component } - else if (i < 192) + else if (i < 192) { // Green to yellow transition palette[i][0] = static_cast((i - 128) * 4); // Red component palette[i][1] = static_cast(255); // Green component palette[i][2] = static_cast(0); // Blue component } - else + else { // Yellow to red transition palette[i][0] = static_cast(255); // Red component @@ -58,9 +61,9 @@ static void create_mandelbrot(imagef& img) } // Iterate over each pixel in the image - for (int px = 0; px < width; px++) + for (int px = 0; px < width; px++) { - for (int py = 0; py < height; py++) + for (int py = 0; py < height; py++) { double x0 = (px - width / 2.0) * 4.0 / width; double y0 = (py - height / 2.0) * 4.0 / height; @@ -71,7 +74,7 @@ static void create_mandelbrot(imagef& img) double x_temp; int iter; - for (iter = 0; iter < max_iter; iter++) + for (iter = 0; iter < max_iter; iter++) { zx_squared = zx * zx; zy_squared = zy * zy; @@ -148,7 +151,7 @@ static bool encode_uastc_ldr() // basis_compress() is a simple wrapper around the basis_compressor_params and basis_compressor classes. void* pKTX2_data = basis_compress( - basist::basis_tex_format::cUASTC4x4, + basist::basis_tex_format::cUASTC_LDR_4x4, source_images, cFlagThreaded | cFlagPrintStats | cFlagDebug | cFlagPrintStatus, 0.0f, &file_size, @@ -173,7 +176,7 @@ static bool encode_uastc_ldr() static bool encode_uastc_hdr() { const uint32_t W = 256, H = 256; - + imagef img(W, H); #if 1 @@ -196,7 +199,7 @@ static bool encode_uastc_hdr() params.m_write_output_basis_or_ktx2_files = true; params.m_out_filename = "test_uastc_hdr.ktx2"; params.m_perceptual = true; - + #if 1 // Create a job pool containing 7 total threads (the calling thread plus 6 additional threads). // A job pool must be created, even if threading is disabled. It's fine to pass in 0 for NUM_THREADS. @@ -219,13 +222,13 @@ static bool encode_uastc_hdr() basisu::basis_compressor::error_code ec = comp.process(); if (ec != basisu::basis_compressor::cECSuccess) return false; - + return true; } // This example function loads a .KTX2 file and then transcodes it to various compressed/uncompressed texture formats. -// It writes .DDS and .ASTC files. -// ARM's astcenc tool can be used to unpack the .ASTC file: +// It writes .DDS and .ASTC files. +// ARM's astcenc tool can be used to unpack the .ASTC file: // astcenc-avx2.exe -dh test_uastc_hdr_astc.astc out.exr static bool transcode_hdr() { @@ -252,10 +255,10 @@ static bool transcode_hdr() // This example only transcodes UASTC HDR textures. if (!transcoder.is_hdr()) return false; - + // Begin transcoding (this will be a no-op with UASTC HDR textures, but you still need to do it. For ETC1S it'll unpack the global codebooks.) transcoder.start_transcoding(); - + // Transcode to BC6H and write a BC6H .DDS file. { gpu_image tex(texture_format::cBC6HUnsigned, width, height); @@ -268,7 +271,7 @@ static bool transcode_hdr() gpu_image_vec tex_vec; tex_vec.push_back(tex); - if (!write_compressed_texture_file("test_uastc_hdr_bc6h.dds", tex_vec, true)) + if (!write_compressed_texture_file("test_uastc_hdr_bc6h.dds", tex_vec, false)) return false; } @@ -423,7 +426,7 @@ const uint32_t NUM_TEST_BLOCKS = (sizeof(g_test_blocks) / sizeof(g_test_blocks[0 static bool block_unpack_and_transcode_example(void) { printf("block_unpack_and_transcode_example:\n"); - + for (uint32_t test_block_iter = 0; test_block_iter < NUM_TEST_BLOCKS; test_block_iter++) { printf("-- Test block %u:\n", test_block_iter); @@ -431,7 +434,7 @@ static bool block_unpack_and_transcode_example(void) const uint8_t* pASTC_blk = &g_test_blocks[test_block_iter * 2 + 0][0]; const uint8_t* pBC6H_blk = &g_test_blocks[test_block_iter * 2 + 1][0]; - // Unpack the physical ASTC block to logical. + // Unpack the physical ASTC block to logical. // Note this is a full ASTC block unpack, and is not specific to UASTC. It does not verify that the block follows the UASTC HDR spec, only ASTC. astc_helpers::log_astc_block log_blk; bool status = astc_helpers::unpack_block(pASTC_blk, log_blk, 4, 4); @@ -470,7 +473,7 @@ static bool block_unpack_and_transcode_example(void) return false; } } // test_block_iter - + printf("Transcode test OK\n"); return true; @@ -492,7 +495,7 @@ static void fuzz_uastc_hdr_transcoder_test() for (uint32_t t = 0; t < NUM_TRIES; t++) { basist::astc_blk astc_blk; - + if (rg.frand(0.0f, 1.0f) < .3f) { // Fully random block @@ -564,6 +567,85 @@ static void fuzz_uastc_hdr_transcoder_test() printf("OK\n"); } +void wrap_image(const image& src, image& dst, int gridX, int gridY, float maxOffset, bool randomize, basisu::rand &rnd) +{ + if (gridX < 1) gridX = 1; + if (gridY < 1) gridY = 1; + + const int vxCountX = gridX + 1; + const int vxCountY = gridY + 1; + const int stride = vxCountX; + + const int w = src.get_width(); + const int h = src.get_height(); + + dst.resize(w, h); + + dst.set_all(g_black_color); + + basisu::vector verts(vxCountX * vxCountY); + basisu::vector uvs(vxCountX * vxCountY); + basisu::vector cols(vxCountX * vxCountY); + + for (int gy = 0; gy <= gridY; ++gy) + { + for (int gx = 0; gx <= gridX; ++gx) + { + float x = (gx / float(gridX)) * (w - 1); + float y = (gy / float(gridY)) * (h - 1); + + float rx = x; + float ry = y; + + if (randomize) + { + rx += rnd.frand(-maxOffset, maxOffset); + ry += rnd.frand(-maxOffset, maxOffset); + } + + verts[gy * stride + gx] = { rx, ry }; + + float u = gx / float(gridX); + float v = gy / float(gridY); + + u = std::max(0.0f, std::min(1.0f, u)); + v = std::max(0.0f, std::min(1.0f, v)); + + uvs[gy * stride + gx] = { u, v }; + + color_rgba c(g_white_color); + + cols[gy * stride + gx] = c; + } + } + + for (int gy = 0; gy < gridY; ++gy) + { + for (int gx = 0; gx < gridX; ++gx) + { + int i0 = gy * stride + gx; + int i1 = i0 + 1; + int i2 = i0 + stride; + int i3 = i2 + 1; + + tri2 tA; + tA.p0 = verts[i0]; tA.p1 = verts[i1]; tA.p2 = verts[i3]; + tA.t0 = uvs[i0]; tA.t1 = uvs[i1]; tA.t2 = uvs[i3]; + tA.c0 = cols[i0]; tA.c1 = cols[i1]; tA.c2 = cols[i3]; + + draw_tri2(dst, &src, tA, randomize); + + tri2 tB; + tB.p0 = verts[i0]; tB.p1 = verts[i3]; tB.p2 = verts[i2]; + tB.t0 = uvs[i0]; tB.t1 = uvs[i3]; tB.t2 = uvs[i2]; + tB.c0 = cols[i0]; tB.c1 = cols[i3]; tB.c2 = cols[i2]; + + draw_tri2(dst, &src, tB, randomize); + } // gx + } // by + +} + enum class codec_class { cETC1S = 0, @@ -571,20 +653,28 @@ enum class codec_class cUASTC_HDR_4x4 = 2, cASTC_HDR_6x6 = 3, cUASTC_HDR_6x6 = 4, + cASTC_LDR = 5, + cXUASTC_LDR = 6, cTOTAL }; -// The main point of this test is to exercise lots of internal compressor code paths, and transcoder code paths. -bool random_compression_fuzz_test() +// The main point of this test is to exercise lots of internal code paths. +bool random_compress_test() { printf("Random XUASTC/ASTC LDR 4x4-12x12 compression test:\n"); - //const uint32_t N = 256; - const uint32_t N = 64; + const uint32_t num_images = 18; + image test_images[num_images + 1]; + + for (uint32_t i = 0; i < num_images; i++) + load_png(fmt_string("../test_files/kodim{02}.png", 1 + i).c_str(), test_images[i]); + + const uint32_t N = 16; + //const uint32_t N = 5000; const uint32_t MAX_WIDTH = 1024, MAX_HEIGHT = 1024; - + basisu::rand rnd; - + float lowest_psnr1 = BIG_FLOAT_VAL, lowest_psnr2 = BIG_FLOAT_VAL; struct result @@ -598,9 +688,11 @@ bool random_compression_fuzz_test() for (uint32_t i = 0; i < N; i++) { - uint32_t seed = 0x2603455 + i; + uint32_t seed = 166136844 + i; - //seed = 23082246; // ETC1S perceptual colorspace error overflow test + //seed = 23082246; // etc1s 1-bit SSE overflow + //seed = 56636601; // UASTC HDR 4x4 assert tol + //seed = 56636744; // HDR 6x6 float overflow fmt_printf("------------------------------ Seed: {}\n", seed); rnd.seed(seed); @@ -609,7 +701,7 @@ bool random_compression_fuzz_test() const uint32_t h = rnd.irand(1, MAX_HEIGHT); const bool mips = rnd.bit(); const bool use_a = rnd.bit(); - + fmt_printf("Trying {}x{}, mips: {}, use_a: {}\n", w, h, mips, use_a); // Chose a random codec/block size to test @@ -618,7 +710,10 @@ bool random_compression_fuzz_test() bool is_hdr = false; uint32_t rnd_codec_class = rnd.irand(0, (uint32_t)codec_class::cTOTAL - 1); - + + // TODO - make this a command line + //rnd_codec_class = rnd.bit() ? (uint32_t)codec_class::cXUASTC_LDR : (uint32_t)codec_class::cASTC_LDR; + //rnd_codec_class = (uint32_t)codec_class::cXUASTC_LDR; //rnd_codec_class = (uint32_t)codec_class::cETC1S; switch (rnd_codec_class) @@ -630,7 +725,7 @@ bool random_compression_fuzz_test() } case (uint32_t)codec_class::cUASTC_LDR_4x4: { - tex_mode = basist::basis_tex_format::cUASTC4x4; + tex_mode = basist::basis_tex_format::cUASTC_LDR_4x4; break; } case (uint32_t)codec_class::cUASTC_HDR_4x4: @@ -647,10 +742,24 @@ bool random_compression_fuzz_test() } case (uint32_t)codec_class::cUASTC_HDR_6x6: { - tex_mode = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + tex_mode = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; is_hdr = true; break; } + case (uint32_t)codec_class::cASTC_LDR: + { + // ASTC LDR 4x4-12x12 + const uint32_t block_variant = rnd.irand(0, astc_helpers::NUM_ASTC_BLOCK_SIZES - 1); + tex_mode = (basist::basis_tex_format)((uint32_t)basist::basis_tex_format::cASTC_LDR_4x4 + block_variant); + break; + } + case (uint32_t)codec_class::cXUASTC_LDR: + { + // XUASTC LDR 4x4-12x12 + const uint32_t block_variant = rnd.irand(0, astc_helpers::NUM_ASTC_BLOCK_SIZES - 1); + tex_mode = (basist::basis_tex_format)((uint32_t)basist::basis_tex_format::cXUASTC_LDR_4x4 + block_variant); + break; + } default: assert(0); tex_mode = basist::basis_tex_format::cETC1S; @@ -658,9 +767,9 @@ bool random_compression_fuzz_test() } fmt_printf("Testing basis_tex_format={}\n", (uint32_t)tex_mode); - + size_t comp_size = 0; - + // Create random LDR source image to compress image src_img; src_img.resize(w, h, w, color_rgba(rnd.byte(), rnd.byte(), rnd.byte(), use_a ? rnd.byte() : 255)); @@ -668,7 +777,7 @@ bool random_compression_fuzz_test() if (rnd.irand(0, 7) >= 1) { const uint32_t nt = rnd.irand(0, 1000); - + for (uint32_t k = 0; k < nt; k++) { color_rgba c(rnd.byte(), rnd.byte(), rnd.byte(), use_a ? rnd.byte() : 255); @@ -680,7 +789,7 @@ bool random_compression_fuzz_test() uint32_t xe = rnd.irand(0, w - 1); if (xs > xe) std::swap(xs, xe); - + uint32_t ys = rnd.irand(0, h - 1); uint32_t ye = rnd.irand(0, h - 1); if (ys > ye) @@ -695,7 +804,7 @@ bool random_compression_fuzz_test() uint32_t ys = rnd.irand(0, h - 1); uint32_t ye = rnd.irand(0, h - 1); - + basisu::draw_line(src_img, xs, ys, xe, ye, c); } else if (r == 6) @@ -712,9 +821,9 @@ bool random_compression_fuzz_test() uint32_t y = rnd.irand(0, h - 1); uint32_t sx = rnd.irand(1, 3); uint32_t sy = rnd.irand(1, 3); - + uint32_t l = rnd.irand(1, 10); - + char buf[32] = {}; for (uint32_t j = 0; j < l; j++) buf[j] = (char)rnd.irand(32, 127); @@ -725,13 +834,13 @@ bool random_compression_fuzz_test() { uint32_t xs = rnd.irand(0, w - 1); uint32_t ys = rnd.irand(0, h - 1); - + uint32_t xl = rnd.irand(1, 100); uint32_t yl = rnd.irand(1, 100); uint32_t xe = minimum(xs + xl - 1, w - 1); uint32_t ye = minimum(ys + yl - 1, h - 1); - + color_rgba cols[4]; cols[0] = c; for (uint32_t j = 1; j < 4; j++) @@ -778,24 +887,101 @@ bool random_compression_fuzz_test() } else src_img(x, y) = q; - } // x + } // x } // y } + else if ((r < 20) && (num_images)) + { + uint32_t image_index = rnd.irand(0, num_images - 1); + + const image& img = test_images[image_index]; + if (img.get_width()) + { + float tw = (float)rnd.irand(1, minimum(128, img.get_width())); + float th = (float)rnd.irand(1, minimum(128, img.get_height())); + + float u = (float)rnd.irand(0, img.get_width() - (int)tw); + float v = (float)rnd.irand(0, img.get_height() - (int)th); + + u /= (float)img.get_width(); + v /= (float)img.get_height(); + + tw /= (float)img.get_width(); + th /= (float)img.get_height(); + + float dx = (float)rnd.irand(0, src_img.get_width() - 1); + float dy = (float)rnd.irand(0, src_img.get_height() - 1); + + float dw = (float)rnd.irand(1, minimum(256, img.get_width())); + float dh = (float)rnd.irand(1, minimum(256, img.get_height())); + + tri2 tri; + tri.p0.set(dx, dy); + tri.t0.set(u, v); + + tri.p1.set(dx + dw, dy); + tri.t1.set(u + tw, v); + + tri.p2.set(dx + dw, dy + dh); + tri.t2.set(u + tw, v + th); + + bool alpha_blend = rnd.bit(); + + if (alpha_blend) + { + tri.c0.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + tri.c1.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + tri.c2.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + } + else + { + tri.c0 = g_white_color; + tri.c1 = g_white_color; + tri.c2 = g_white_color; + } + + draw_tri2(src_img, &img, tri, alpha_blend); + + tri.p0.set(dx, dy); + tri.t0.set(u, v); + + tri.p1.set(dx + dw, dy + dh); + tri.t1.set(u + tw, v + th); + tri.c1 = tri.c2; + + tri.p2.set(dx, dy + dh); + tri.t2.set(u, v + th); + tri.c2.set(rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(100, 255), rnd.irand(1, 255)); + + draw_tri2(src_img, &img, tri, alpha_blend); + } + } else { src_img(rnd.irand(0, w - 1), rnd.irand(0, h - 1)) = c; } } } - + if ((use_a) && (rnd.irand(0, 3) >= 2)) { const uint32_t nt = rnd.irand(0, 1000); - + for (uint32_t k = 0; k < nt; k++) src_img(rnd.irand(0, w - 1), rnd.irand(0, h - 1)).a = rnd.byte(); } + if (rnd.bit()) + { + int gridX = rnd.irand(8, 24); + int gridY = rnd.irand(8, 24); + float maxOffset = rnd.frand(0.0f, (float)maximum(gridX, gridY)); + + image tmp_img; + wrap_image(src_img, tmp_img, gridX, gridY, maxOffset, true, rnd); + src_img.swap(tmp_img); + } + if (!use_a) { for (uint32_t y = 0; y < h; y++) @@ -805,12 +991,12 @@ bool random_compression_fuzz_test() //save_png("test.png", src_img); //fmt_printf("Has alpha: {}\n", src_img.has_alpha()); - + // Choose randomized codec parameters uint32_t flags = cFlagPrintStats | cFlagValidateOutput | cFlagPrintStatus; - - //flags |= cFlagDebug; - + + flags |= cFlagDebug; + flags |= cFlagThreaded; if (rnd.bit()) @@ -826,16 +1012,16 @@ bool random_compression_fuzz_test() flags |= cFlagREC2020; float quality = 0.0f; - + switch (rnd_codec_class) { case (uint32_t)codec_class::cETC1S: { // ETC1S - + // Choose random ETC1S quality level flags |= rnd.irand(1, 255); - + break; } case (uint32_t)codec_class::cUASTC_LDR_4x4: @@ -846,7 +1032,6 @@ bool random_compression_fuzz_test() { // Choose random RDO lambda quality = rnd.frand(0.0, 10.0f); - flags |= cFlagUASTCRDO; } // Choose random effort level @@ -857,7 +1042,7 @@ bool random_compression_fuzz_test() case (uint32_t)codec_class::cUASTC_HDR_4x4: { // UASTC HDR 4x4 - + // Choose random effort level. flags |= rnd.irand(uastc_hdr_4x4_codec_options::cMinLevel, uastc_hdr_4x4_codec_options::cMaxLevel); @@ -867,7 +1052,7 @@ bool random_compression_fuzz_test() case (uint32_t)codec_class::cUASTC_HDR_6x6: { // RDO ASTC HDR 6x6 or UASTC HDR 6x6 - + // Chose random effort level flags |= rnd.irand(0, astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL); @@ -879,12 +1064,33 @@ bool random_compression_fuzz_test() break; } + case (uint32_t)codec_class::cASTC_LDR: + case (uint32_t)codec_class::cXUASTC_LDR: + { + // ASTC/XUASTC LDR 4x4-12x12 + + // Choose random profile + uint32_t xuastc_ldr_syntax = rnd.irand(0, (uint32_t)basist::astc_ldr_t::xuastc_ldr_syntax::cTotal - 1); + flags |= (xuastc_ldr_syntax << cFlagXUASTCLDRSyntaxShift); + + // Choose random effort + uint32_t effort = rnd.irand(basisu::astc_ldr::EFFORT_LEVEL_MIN, basisu::astc_ldr::EFFORT_LEVEL_MAX); + flags |= effort; + + // Choose random weight grid DCT quality + quality = (float)rnd.frand(1.0f, 100.0f); + + if (rnd.irand(0, 7) == 0) + quality = 0.0f; // sometimes disable DCT + + break; + } default: { assert(0); } } - + void* pComp_data = nullptr; image_stats stats; @@ -892,7 +1098,7 @@ bool random_compression_fuzz_test() { basisu::vector hdr_source_images; imagef hdr_src_img(src_img.get_width(), src_img.get_height()); - + const float max_y = rnd.frand(.000125f, 30000.0f) / 255.0f; for (uint32_t y = 0; y < src_img.get_height(); y++) @@ -907,7 +1113,7 @@ bool random_compression_fuzz_test() } //write_exr("test.exr", hdr_src_img, 3, 0); - + hdr_source_images.push_back(hdr_src_img); pComp_data = basisu::basis_compress(tex_mode, hdr_source_images, flags, quality, &comp_size, &stats); } @@ -917,7 +1123,8 @@ bool random_compression_fuzz_test() ldr_source_images.push_back(src_img); //save_png("test.png", src_img); - + //save_png(fmt_string("test_{}.png", seed), src_img); + pComp_data = basisu::basis_compress(tex_mode, ldr_source_images, flags, quality, &comp_size, &stats); } @@ -941,39 +1148,66 @@ bool random_compression_fuzz_test() psnr2 }); } // i - + printf("PSNR Results:\n"); - + for (uint32_t i = 0; i < results.size(); i++) fmt_printf("{},{},{},{}\n", results[i].m_seed, (uint32_t)results[i].m_fmt, results[i].m_psnr1, results[i].m_psnr2); - + printf("\n"); for (uint32_t i = 0; i < results.size(); i++) fmt_printf("seed={} tex_mode={}, psnr1={}, psnr2={}\n", results[i].m_seed, (uint32_t)results[i].m_fmt, results[i].m_psnr1, results[i].m_psnr2); - + // Success here is essentially not crashing or asserting or SAN'ing earlier printf("Success\n"); - + return true; } +#ifdef FORCE_SAN_FAILURE +static void force_san_failure() +{ + // Purposely do things that should trigger the address sanitizer + int arr[5] = { 0, 1, 2, 3, 4 }; + printf("Out of bounds element: %d\n", arr[10]); + + //uint8_t* p = (uint8_t *)malloc(10); + //p[10] = 99; + + //uint8_t* p = (uint8_t *)malloc(10); + //free(p); + //p[0] = 99; +} +#endif // FORCE_SAN_FAILURE + int main(int arg_c, char* arg_v[]) { BASISU_NOTE_UNUSED(arg_c); BASISU_NOTE_UNUSED(arg_v); +#if defined(DEBUG) | defined(_DEBUG) + printf("DEBUG\n"); +#endif +#ifdef __SANITIZE_ADDRESS__ + printf("__SANITIZE_ADDRESS__\n"); +#endif + +#ifdef FORCE_SAN_FAILURE + force_san_failure(); +#endif + #if USE_ENCODER basisu_encoder_init(USE_OPENCL, false); - if (!random_compression_fuzz_test()) + if (!random_compress_test()) return EXIT_FAILURE; - + if (!block_unpack_and_transcode_example()) return EXIT_FAILURE; fuzz_uastc_hdr_transcoder_test(); - + if (!encode_etc1s()) { fprintf(stderr, "encode_etc1s() failed!\n"); diff --git a/example/example.vcxproj b/example/example.vcxproj index bd10772..d33702a 100644 --- a/example/example.vcxproj +++ b/example/example.vcxproj @@ -38,40 +38,40 @@ Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application true Unicode - v143 + v145 Application false true Unicode - v143 + v145 Application false true Unicode - v143 + v145 @@ -100,10 +100,21 @@ $(SolutionDir)\bin\ - + + $(SolutionDir)\bin\ + $(SolutionDir)\bin\ + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + + + $(SolutionDir)\bin\ + Level4 @@ -112,7 +123,7 @@ true ..\OpenCL stdcpp17 - StreamingSIMDExtensions2 + AdvancedVectorExtensions Console @@ -130,7 +141,7 @@ WIN32;NDEBUG;_HAS_EXCEPTIONS=0;_CONSOLE;%(PreprocessorDefinitions);BASISU_SUPPORT_SSE=1;BASISU_SUPPORT_OPENCL=1 true ..\OpenCL - StreamingSIMDExtensions2 + AdvancedVectorExtensions false false stdcpp17 @@ -152,8 +163,8 @@ StreamingSIMDExtensions2 false ..\OpenCL - Level4 stdcpp17 + Level4 Console @@ -243,4 +254,4 @@ - + \ No newline at end of file diff --git a/transcoder/basisu.h b/transcoder/basisu.h index 6e2efed..d4e339e 100644 --- a/transcoder/basisu.h +++ b/transcoder/basisu.h @@ -1,5 +1,5 @@ // basisu.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,12 +15,16 @@ // limitations under the License. #pragma once +#ifndef BASISD_SUPPORT_XUASTC +#define BASISD_SUPPORT_XUASTC 1 +#endif + #ifdef _MSC_VER #pragma warning (disable : 4201) #pragma warning (disable : 4127) // warning C4127: conditional expression is constant #pragma warning (disable : 4530) // C++ exception handler used, but unwind semantics are not enabled. - + #endif // _MSC_VER #include @@ -40,9 +44,11 @@ #include #include #include +#include #include "basisu_containers.h" +// We never use min/max macros, slam them to off. #ifdef max #undef max #endif @@ -57,6 +63,7 @@ // Set to one to enable debug printf()'s when any errors occur, for development/debugging. Especially useful for WebGL development. #ifndef BASISU_FORCE_DEVEL_MESSAGES +// Do not check in as 1! #define BASISU_FORCE_DEVEL_MESSAGES 0 #endif @@ -93,6 +100,7 @@ namespace basisu typedef basisu::vector int_vec; typedef basisu::vector bool_vec; typedef basisu::vector float_vec; + typedef basisu::vector double_vec; void enable_debug_printf(bool enabled); void debug_printf(const char *pFmt, ...); @@ -109,14 +117,14 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif - + template inline void clear_obj(T& obj) { memset((void *)&obj, 0, sizeof(obj)); } #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop -#endif +#endif constexpr double cPiD = 3.14159265358979323846264338327950288; constexpr float REALLY_SMALL_FLOAT_VAL = .000000125f; @@ -124,7 +132,7 @@ namespace basisu constexpr float BIG_FLOAT_VAL = 1e+30f; template inline T0 lerp(T0 a, T0 b, T1 c) { return a + (b - a) * c; } - + inline float clampf(float value, float low, float high) { if (value < low) value = low; else if (value > high) value = high; return value; } inline float saturate(float value) { return clampf(value, 0, 1.0f); } inline uint8_t minimumub(uint8_t a, uint8_t b) { return (a < b) ? a : b; } @@ -141,6 +149,41 @@ namespace basisu template inline T square(T a) { return a * a; } template inline T sign(T a) { return (a < 0) ? (T)-1 : ((a == 0) ? (T)0 : (T)1); } + inline int imod(int i, int d) + { + assert(i != INT_MIN); + + if (i >= 0) + return i % d; + + int r = (-i) % d; + return (r == 0) ? 0 : d - r; + } + + inline uint8_t safe_cast_uint8(uint32_t x) + { + assert(x <= UINT8_MAX); + return (uint8_t)x; + } + + inline int8_t safe_cast_int8(int32_t x) + { + assert((x >= INT8_MIN) && (x <= INT8_MAX)); + return (int8_t)x; + } + + inline uint16_t safe_cast_uint16(uint32_t x) + { + assert(x <= UINT16_MAX); + return (uint16_t)x; + } + + inline int16_t safe_cast_int16(int32_t x) + { + assert((x >= INT16_MIN) && (x <= INT16_MAX)); + return (int16_t)x; + } + inline bool equal_tol(float a, float b, float t) { return fabsf(a - b) <= ((maximum(fabsf(a), fabsf(b)) + 1.0f) * t); } inline bool equal_tol(double a, double b, double t) { return fabs(a - b) <= ((maximum(fabs(a), fabs(b)) + 1.0f) * t); } @@ -161,27 +204,35 @@ namespace basisu temp = 0; return temp; } - + inline uint32_t iabs(int32_t i) { return (i < 0) ? static_cast(-i) : static_cast(i); } inline uint64_t iabs64(int64_t i) { return (i < 0) ? static_cast(-i) : static_cast(i); } - template inline void clear_vector(T &vec) { vec.erase(vec.begin(), vec.end()); } + template inline void clear_vector(T &vec) { vec.erase(vec.begin(), vec.end()); } template inline typename T::value_type *enlarge_vector(T &vec, size_t n) { size_t cs = vec.size(); vec.resize(cs + n); return &vec[cs]; } inline bool is_pow2(uint32_t x) { return x && ((x & (x - 1U)) == 0U); } inline bool is_pow2(uint64_t x) { return x && ((x & (x - 1U)) == 0U); } + template inline T range_check(T v, T minv, T maxv) { assert(v >= minv && v <= maxv); BASISU_NOTE_UNUSED(minv); BASISU_NOTE_UNUSED(maxv); return v; } + template inline T range_check(T v, T maxv) { assert(v <= maxv); BASISU_NOTE_UNUSED(maxv); return v; } + template inline T open_range_check(T v, T minv, T maxv) { assert(v >= minv && v < maxv); BASISU_NOTE_UNUSED(minv); BASISU_NOTE_UNUSED(maxv); return v; } template inline T open_range_check(T v, T maxv) { assert(v < maxv); BASISU_NOTE_UNUSED(maxv); return v; } // Open interval - inline bool in_bounds(int v, int l, int h) + inline bool is_in_bounds(int v, int l, int h) { return (v >= l) && (v < h); } // Closed interval - inline bool in_range(int v, int l, int h) + inline bool is_in_range(int v, int l, int h) + { + return (v >= l) && (v <= h); + } + + inline bool is_in_range(float v, float l, float h) { return (v >= l) && (v <= h); } @@ -192,7 +243,7 @@ namespace basisu inline uint32_t get_bit(uint32_t src, int ndx) { - assert(in_bounds(ndx, 0, 32)); + assert(is_in_bounds(ndx, 0, 32)); return (src >> ndx) & 1; } @@ -204,7 +255,7 @@ namespace basisu inline uint32_t get_bits(uint32_t val, int low, int high) { const int num_bits = (high - low) + 1; - assert(in_range(num_bits, 1, 32)); + assert(is_in_range(num_bits, 1, 32)); val >>= low; if (num_bits != 32) @@ -213,8 +264,8 @@ namespace basisu return val; } - template inline void append_vector(T &vec, const R *pObjs, size_t n) - { + template inline void append_vector(T &vec, const R *pObjs, size_t n) + { if (n) { if (vec.size()) @@ -265,7 +316,7 @@ namespace basisu for (size_t i = 0; i < vec.size(); i++) vec[i] = obj; } - + inline uint64_t read_be64(const void *p) { uint64_t val = 0; @@ -307,6 +358,14 @@ namespace basisu return (m != 0) ? (y - m) : m; } + inline float posmodf(float x, float y) + { + float m = fmodf(x, y); + if (m < 0.0f) + m += y; + return m; + } + inline bool do_excl_ranges_overlap(int la, int ha, int lb, int hb) { assert(la < ha && lb < hb); @@ -331,7 +390,7 @@ namespace basisu pBytes[2] = (uint8_t)(val >> 16U); pBytes[3] = (uint8_t)(val >> 24U); } - + // Always little endian 1-8 byte unsigned int template struct packed_uint @@ -341,21 +400,21 @@ namespace basisu inline packed_uint() { static_assert(NumBytes <= sizeof(uint64_t), "Invalid NumBytes"); } inline packed_uint(uint64_t v) { *this = v; } inline packed_uint(const packed_uint& other) { *this = other; } - - inline packed_uint& operator= (uint64_t v) - { + + inline packed_uint& operator= (uint64_t v) + { // TODO: Add assert on truncation? - for (uint32_t i = 0; i < NumBytes; i++) - m_bytes[i] = static_cast(v >> (i * 8)); - return *this; + for (uint32_t i = 0; i < NumBytes; i++) + m_bytes[i] = static_cast(v >> (i * 8)); + return *this; } - inline packed_uint& operator= (const packed_uint& rhs) - { - memcpy(m_bytes, rhs.m_bytes, sizeof(m_bytes)); + inline packed_uint& operator= (const packed_uint& rhs) + { + memcpy(m_bytes, rhs.m_bytes, sizeof(m_bytes)); return *this; } - + inline uint64_t get_uint64() const { // Some compilers may warn about this code. It clearly cannot access beyond the end of the m_bytes struct here. @@ -411,7 +470,7 @@ namespace basisu static_assert(NumBytes <= sizeof(uint32_t), "packed_uint too large to use get_uint32"); return static_cast(get_uint64()); } - + inline operator uint32_t() const { static_assert(NumBytes <= sizeof(uint32_t), "packed_uint too large to use operator uint32_t"); @@ -421,14 +480,14 @@ namespace basisu enum eZero { cZero }; enum eNoClamp { cNoClamp }; - + // Rice/Huffman entropy coding - + // This is basically Deflate-style canonical Huffman, except we allow for a lot more symbols. enum { - cHuffmanMaxSupportedCodeSize = 16, cHuffmanMaxSupportedInternalCodeSize = 31, - cHuffmanFastLookupBits = 10, + cHuffmanMaxSupportedCodeSize = 16, cHuffmanMaxSupportedInternalCodeSize = 31, + cHuffmanFastLookupBits = 10, cHuffmanMaxSymsLog2 = 14, cHuffmanMaxSyms = 1 << cHuffmanMaxSymsLog2, // Small zero runs @@ -454,13 +513,13 @@ namespace basisu enum class texture_format { cInvalidTextureFormat = -1, - + // Block-based formats cETC1, // ETC1 cETC1S, // ETC1 (subset: diff colors only, no subblocks) cETC2_RGB, // ETC2 color block (basisu doesn't support ETC2 planar/T/H modes - just basic ETC1) cETC2_RGBA, // ETC2 EAC alpha block followed by ETC2 color block - cETC2_ALPHA, // ETC2 EAC alpha block + cETC2_ALPHA, // ETC2 EAC alpha block cBC1, // DXT1 cBC3, // DXT5 (BC4/DXT5A block followed by a BC1/DXT1 block) cBC4, // DXT5A @@ -479,11 +538,11 @@ namespace basisu cPVRTC2_4_RGBA, cETC2_R11_EAC, cETC2_RG11_EAC, - cUASTC4x4, + cUASTC4x4, cUASTC_HDR_4x4, cBC1_NV, cBC1_AMD, - + // Uncompressed/raw pixels cRGBA32, cRGB565, @@ -492,9 +551,89 @@ namespace basisu cABGR4444, cRGBA_HALF, cRGB_HALF, - cRGB_9E5 + cRGB_9E5, + + // All remaining ASTC LDR block size variants (other than 4x4 which is above). There are 14 total ASTC block sizes, including 4x4. + cASTC_LDR_5x4, + cASTC_LDR_5x5, + cASTC_LDR_6x5, + cASTC_LDR_6x6, + cASTC_LDR_8x5, + cASTC_LDR_8x6, + cASTC_LDR_10x5, + cASTC_LDR_10x6, + cASTC_LDR_8x8, + cASTC_LDR_10x8, + cASTC_LDR_10x10, + cASTC_LDR_12x10, + cASTC_LDR_12x12 }; + inline bool is_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_HDR_4x4: + case texture_format::cASTC_HDR_6x6: + case texture_format::cASTC_LDR_4x4: + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: + return true; + default: + break; + } + return false; + } + + inline bool is_hdr_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_HDR_4x4: + case texture_format::cASTC_HDR_6x6: + return true; + default: + break; + } + return false; + } + + inline bool is_ldr_astc(texture_format fmt) + { + switch (fmt) + { + case texture_format::cASTC_LDR_4x4: + case texture_format::cASTC_LDR_5x4: + case texture_format::cASTC_LDR_5x5: + case texture_format::cASTC_LDR_6x5: + case texture_format::cASTC_LDR_6x6: + case texture_format::cASTC_LDR_8x5: + case texture_format::cASTC_LDR_8x6: + case texture_format::cASTC_LDR_10x5: + case texture_format::cASTC_LDR_10x6: + case texture_format::cASTC_LDR_8x8: + case texture_format::cASTC_LDR_10x8: + case texture_format::cASTC_LDR_10x10: + case texture_format::cASTC_LDR_12x10: + case texture_format::cASTC_LDR_12x12: + return true; + default: + break; + } + return false; + } + inline bool is_uncompressed_texture_format(texture_format fmt) { switch (fmt) @@ -555,7 +694,7 @@ namespace basisu default: break; } - + // Everything else is 16 bytes/block. return 16; } @@ -575,10 +714,21 @@ namespace basisu switch (fmt) { - case texture_format::cFXT1_RGB: - return 8; - case texture_format::cASTC_HDR_6x6: - return 6; + case texture_format::cFXT1_RGB: return 8; + case texture_format::cASTC_HDR_6x6: return 6; + case texture_format::cASTC_LDR_5x4: return 5; + case texture_format::cASTC_LDR_5x5: return 5; + case texture_format::cASTC_LDR_6x5: return 6; + case texture_format::cASTC_LDR_6x6: return 6; + case texture_format::cASTC_LDR_8x5: return 8; + case texture_format::cASTC_LDR_8x6: return 8; + case texture_format::cASTC_LDR_10x5: return 10; + case texture_format::cASTC_LDR_10x6: return 10; + case texture_format::cASTC_LDR_8x8: return 8; + case texture_format::cASTC_LDR_10x8: return 10; + case texture_format::cASTC_LDR_10x10: return 10; + case texture_format::cASTC_LDR_12x10: return 12; + case texture_format::cASTC_LDR_12x12: return 12; default: break; } @@ -591,8 +741,19 @@ namespace basisu switch (fmt) { - case texture_format::cASTC_HDR_6x6: - return 6; + case texture_format::cASTC_HDR_6x6: return 6; + case texture_format::cASTC_LDR_5x5: return 5; + case texture_format::cASTC_LDR_6x5: return 5; + case texture_format::cASTC_LDR_6x6: return 6; + case texture_format::cASTC_LDR_8x5: return 5; + case texture_format::cASTC_LDR_8x6: return 6; + case texture_format::cASTC_LDR_10x5: return 5; + case texture_format::cASTC_LDR_10x6: return 6; + case texture_format::cASTC_LDR_8x8: return 8; + case texture_format::cASTC_LDR_10x8: return 8; + case texture_format::cASTC_LDR_10x10: return 10; + case texture_format::cASTC_LDR_12x10: return 10; + case texture_format::cASTC_LDR_12x12: return 12; default: break; } @@ -623,5 +784,38 @@ namespace basisu { return !is_hdr_texture_format(fmt); } + + inline texture_format get_astc_ldr_texture_format(uint32_t width, uint32_t height) + { +#define BU_ASTC_LDR_MATCH_BLOCK_DIM(x, y, f) if ((width == (x)) && (height == (y))) return (f); + BU_ASTC_LDR_MATCH_BLOCK_DIM(4, 4, texture_format::cASTC_LDR_4x4); + BU_ASTC_LDR_MATCH_BLOCK_DIM(5, 4, texture_format::cASTC_LDR_5x4); + BU_ASTC_LDR_MATCH_BLOCK_DIM(5, 5, texture_format::cASTC_LDR_5x5); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(6, 5, texture_format::cASTC_LDR_6x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(6, 6, texture_format::cASTC_LDR_6x6); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 5, texture_format::cASTC_LDR_8x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 6, texture_format::cASTC_LDR_8x6); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 5, texture_format::cASTC_LDR_10x5); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 6, texture_format::cASTC_LDR_10x6); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(8, 8, texture_format::cASTC_LDR_8x8); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 8, texture_format::cASTC_LDR_10x8); + BU_ASTC_LDR_MATCH_BLOCK_DIM(10, 10, texture_format::cASTC_LDR_10x10); + + BU_ASTC_LDR_MATCH_BLOCK_DIM(12, 10, texture_format::cASTC_LDR_12x10); + BU_ASTC_LDR_MATCH_BLOCK_DIM(12, 12, texture_format::cASTC_LDR_12x12); +#undef BU_ASTC_LDR_MATCH_BLOCK_DIM + + return texture_format::cInvalidTextureFormat; + } + + inline bool is_valid_astc_block_size(uint32_t width, uint32_t height) + { + return get_astc_ldr_texture_format(width, height) != texture_format::cInvalidTextureFormat; + } + } // namespace basisu + diff --git a/transcoder/basisu_astc_hdr_core.h b/transcoder/basisu_astc_hdr_core.h index 8e63f72..143e29e 100644 --- a/transcoder/basisu_astc_hdr_core.h +++ b/transcoder/basisu_astc_hdr_core.h @@ -1,6 +1,5 @@ // File: basisu_astc_hdr_core.h #pragma once -#include "basisu_astc_helpers.h" namespace basist { @@ -109,7 +108,10 @@ namespace basist const uint32_t TOTAL_BLOCK_MODE_DECS = 75; extern const block_mode_desc g_block_mode_descs[TOTAL_BLOCK_MODE_DECS]; - void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk); + const uint32_t UASTC_6x6_HDR_SIG0 = 0xABCD; // original release + const uint32_t UASTC_6x6_HDR_SIG1 = 0xABCE; // 2x2->4x4 weight grid upsampling change, not backwards compatible with older decoders + + void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk, bool orig_behavior); enum class encoding_type { @@ -181,7 +183,7 @@ namespace basist bool m_hq_ls; bool m_brute_force_weight4_assignment; - + fast_bc6h_params() { init(); @@ -203,3 +205,4 @@ namespace basist } // namespace astc_6x6_hdr } // namespace basist + diff --git a/transcoder/basisu_astc_helpers.h b/transcoder/basisu_astc_helpers.h index 3791805..32b1608 100644 --- a/transcoder/basisu_astc_helpers.h +++ b/transcoder/basisu_astc_helpers.h @@ -10,12 +10,22 @@ namespace astc_helpers { - const uint32_t MAX_WEIGHT_VALUE = 64; // grid texel weights must range from [0,64] const uint32_t MIN_GRID_DIM = 2; // the minimum dimension of a block's weight grid const uint32_t MIN_BLOCK_DIM = 4, MAX_BLOCK_DIM = 12; // the valid block dimensions in texels + const uint32_t MAX_BLOCK_PIXELS = MAX_BLOCK_DIM * MAX_BLOCK_DIM; const uint32_t MAX_GRID_WEIGHTS = 64; // a block may have a maximum of 64 weight grid values - const uint32_t NUM_MODE11_ENDPOINTS = 6, NUM_MODE7_ENDPOINTS = 4; + const uint32_t MAX_CEM_ENDPOINT_VALS = 8; // see Table 94. ASTC LDR/HDR color endpoint modes (max 8 values to encode any CEM, minimum 2) + // The number of BISE values needed to encode endpoints for each CEM. + const uint32_t NUM_MODE0_ENDPOINTS = 2, NUM_MODE4_ENDPOINTS = 4; + const uint32_t NUM_MODE6_ENDPOINTS = 4, NUM_MODE8_ENDPOINTS = 6, NUM_MODE9_ENDPOINTS = 6; // LDR RGB + const uint32_t NUM_MODE10_ENDPOINTS = 6, NUM_MODE12_ENDPOINTS = 8, NUM_MODE13_ENDPOINTS = 8; // LDR RGBA + const uint32_t NUM_MODE11_ENDPOINTS = 6, NUM_MODE7_ENDPOINTS = 4; // hdr + + const uint32_t MAX_WEIGHTS = 32; // max supported # of weights (or "selectors") in any mode, i.e. the max # of colors per endpoint pair + const uint32_t MAX_WEIGHT_INTERPOLANT_VALUE = 64; // grid texel weights must range from [0,64], i.e. the weight interpolant range is [0,64] + + // 14 unique block dimensions supported by ASTC static const uint32_t NUM_ASTC_BLOCK_SIZES = 14; extern const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2]; @@ -70,6 +80,29 @@ namespace astc_helpers const uint32_t TOTAL_ISE_RANGES = 21; + enum + { + cBLOCK_SIZE_4x4 = 0, // 16 samples + cBLOCK_SIZE_5x4 = 1, // 20 samples + cBLOCK_SIZE_5x5 = 2, // 25 samples + cBLOCK_SIZE_6x5 = 3, // 30 samples + + cBLOCK_SIZE_6x6 = 4, // 36 samples + cBLOCK_SIZE_8x5 = 5, // 40 samples + cBLOCK_SIZE_8x6 = 6, // 48 samples + cBLOCK_SIZE_10x5 = 7, // 50 samples + + cBLOCK_SIZE_10x6 = 8, // 60 samples + cBLOCK_SIZE_8x8 = 9, // 64 samples + cBLOCK_SIZE_10x8 = 10, // 80 samples + cBLOCK_SIZE_10x10 = 11, // 100 samples + + cBLOCK_SIZE_12x10 = 12, // 120 samples + cBLOCK_SIZE_12x12 = 13, // 144 samples + + cTOTAL_BLOCK_SIZES = 14 + }; + // Valid endpoint ISE ranges const uint32_t FIRST_VALID_ENDPOINT_ISE_RANGE = BISE_6_LEVELS; // 4 const uint32_t LAST_VALID_ENDPOINT_ISE_RANGE = BISE_256_LEVELS; // 20 @@ -79,11 +112,11 @@ namespace astc_helpers const uint32_t FIRST_VALID_WEIGHT_ISE_RANGE = BISE_2_LEVELS; // 0 const uint32_t LAST_VALID_WEIGHT_ISE_RANGE = BISE_32_LEVELS; // 11 const uint32_t TOTAL_WEIGHT_ISE_RANGES = LAST_VALID_WEIGHT_ISE_RANGE - FIRST_VALID_WEIGHT_ISE_RANGE + 1; - + // The ISE range table. extern const int8_t g_ise_range_table[TOTAL_ISE_RANGES][3]; // 0=bits (0 to 8), 1=trits (0 or 1), 2=quints (0 or 1) - // Possible Color Component Select values, used in dual plane mode. + // Possible Color Component Select values, used in dual plane mode. // The CCS component will be interpolated using the 2nd weight plane. enum ccs { @@ -92,7 +125,7 @@ namespace astc_helpers CCS_RGA_B = 2, CCS_RGB_A = 3 }; - + struct astc_block { uint32_t m_vals[4]; @@ -106,14 +139,14 @@ namespace astc_helpers struct log_astc_block { bool m_error_flag; - + bool m_solid_color_flag_ldr, m_solid_color_flag_hdr; uint8_t m_user_mode; // user defined value, not used in this module - + // Rest is only valid if !m_solid_color_flag_ldr && !m_solid_color_flag_hdr uint8_t m_grid_width, m_grid_height; // weight grid dimensions, not the dimension of the block - + bool m_dual_plane; uint8_t m_weight_ise_range; // 0-11 @@ -123,16 +156,16 @@ namespace astc_helpers uint8_t m_num_partitions; // or the # of subsets, 1-4 (1-3 if dual plane mode) uint16_t m_partition_id; // 10-bits, must be 0 if m_num_partitions==1 - + uint8_t m_color_endpoint_modes[MAX_PARTITIONS]; // each subset's CEM's - + union { // ISE weight grid values. In dual plane mode, the order is p0,p1, p0,p1, etc. uint8_t m_weights[MAX_GRID_WEIGHTS]; uint16_t m_solid_color[4]; }; - + // ISE endpoint values // Endpoint order examples: // 1 subset LA : LL0 LH0 AL0 AH0 @@ -142,13 +175,13 @@ namespace astc_helpers // 2 subset RGB : RL0 RH0 GL0 GH0 BL0 BH0 RL1 RH1 GL1 GH1 BL1 BH1 // 2 subset RGBA : RL0 RH0 GL0 GH0 BL0 BH0 AL0 AH0 RL1 RH1 GL1 GH1 BL1 BH1 AL1 AH1 uint8_t m_endpoints[MAX_ENDPOINTS]; - + void clear() { memset(this, 0, sizeof(*this)); } }; - + // Open interval inline int bounds_check(int v, int l, int h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } inline uint32_t bounds_check(uint32_t v, uint32_t l, uint32_t h) { (void)v; (void)l; (void)h; assert(v >= l && v < h); return v; } @@ -166,8 +199,8 @@ namespace astc_helpers } // Returns the number of levels in the given ISE range. - inline uint32_t get_ise_levels(uint32_t ise_range) - { + inline uint32_t get_ise_levels(uint32_t ise_range) + { assert(ise_range < TOTAL_ISE_RANGES); return (1 + 2 * g_ise_range_table[ise_range][1] + 4 * g_ise_range_table[ise_range][2]) << g_ise_range_table[ise_range][0]; } @@ -180,10 +213,10 @@ namespace astc_helpers total_bits += (g_ise_range_table[range][2] * 7 * count + 2) / 3; return total_bits; } - + inline uint32_t weight_interpolate(uint32_t l, uint32_t h, uint32_t w) { - assert(w <= MAX_WEIGHT_VALUE); + assert(w <= MAX_WEIGHT_INTERPOLANT_VALUE); return (l * (64 - w) + h * w + 32) >> 6; } @@ -198,9 +231,15 @@ namespace astc_helpers inline pack_stats() { clear(); } inline void clear() { memset(this, 0, sizeof(*this)); } }; - + + enum + { + cValidateEarlyOutAtEndpointISEChecks = 1, + cValidateSkipFinalEndpointWeightPacking = 2, + }; + // Packs a logical to physical ASTC block. Note this does not validate the block's dimensions (use is_valid_block_size()), just the grid dimensions. - bool pack_astc_block(astc_block &phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range = nullptr, pack_stats *pStats = nullptr); + bool pack_astc_block(astc_block &phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range = nullptr, pack_stats *pStats = nullptr, uint32_t validate_flags = 0); // Pack LDR void extent (really solid color) blocks. For LDR, pass in (val | (val << 8)) for each component. void pack_void_extent_ldr(astc_block& blk, uint16_t r, uint16_t g, uint16_t b, uint16_t a, pack_stats *pStats = nullptr); @@ -209,10 +248,10 @@ namespace astc_helpers void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats* pStats = nullptr); // These helpers are all quite slow, but are useful for table preparation. - + // Dequantizes ISE encoded endpoint val to [0,255] uint32_t dequant_bise_endpoint(uint32_t val, uint32_t ise_range); // ISE ranges 4-11 - + // Dequantizes ISE encoded weight val to [0,64] uint32_t dequant_bise_weight(uint32_t val, uint32_t ise_range); // ISE ranges 0-10 @@ -231,12 +270,21 @@ namespace astc_helpers bool is_cem_ldr(uint32_t mode); inline bool is_cem_hdr(uint32_t mode) { return !is_cem_ldr(mode); } + bool does_cem_have_alpha(uint32_t mode); + // True if the passed in dimensions are a valid ASTC block size. There are 14 supported configs, from 4x4 (8bpp) to 12x12 (.89bpp). bool is_valid_block_size(uint32_t w, uint32_t h); + + // w/h must be a valid ASTC block size, or it returns cBLOCK_SIZE_4x4 + uint32_t get_block_size_index(uint32_t w, uint32_t h); + + float get_bitrate_from_block_size(uint32_t w, uint32_t h); + + uint32_t get_texel_partition_from_table(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t subsets, uint32_t x, uint32_t y); bool block_has_any_hdr_cems(const log_astc_block& log_blk); bool block_has_any_ldr_cems(const log_astc_block& log_blk); - + // Returns the # of endpoint values for the given CEM. inline uint32_t get_num_cem_values(uint32_t cem) { assert(cem <= 15); return 2 + 2 * (cem >> 2); } @@ -245,17 +293,28 @@ namespace astc_helpers basisu::vector m_val_to_ise; // [0-255] or [0-64] value to nearest ISE symbol, array size is [256] or [65] basisu::vector m_ISE_to_val; // ASTC encoded ISE symbol to [0,255] or [0,64] value, [levels] basisu::vector m_ISE_to_rank; // returns the level rank index given an ISE symbol, [levels] - basisu::vector m_rank_to_ISE; // returns the ISE symbol given a level rank, inverse of pISE_to_rank, [levels] + basisu::vector m_rank_to_ISE; // returns the ISE symbol given a level rank, inverse of pISE_to_rank, [levels] - void init(bool weight_flag, uint32_t num_levels, bool init_rank_tabs) + void init(bool weight_flag, uint32_t num_levels) { - m_val_to_ise.resize(weight_flag ? (MAX_WEIGHT_VALUE + 1) : 256); + m_val_to_ise.resize(weight_flag ? (MAX_WEIGHT_INTERPOLANT_VALUE + 1) : 256); m_ISE_to_val.resize(num_levels); - if (init_rank_tabs) - { - m_ISE_to_rank.resize(num_levels); - m_rank_to_ISE.resize(num_levels); - } + m_ISE_to_rank.resize(num_levels); + m_rank_to_ISE.resize(num_levels); + } + + uint32_t get_rank_to_val(uint32_t rank) const + { + const uint32_t ise = m_rank_to_ISE[rank]; + const uint32_t val = m_ISE_to_val[ise]; + return val; + } + + uint32_t get_val_to_rank(uint32_t val) + { + const uint32_t ise = m_val_to_ise[val]; + const uint32_t rank = m_ISE_to_rank[ise]; + return rank; } }; @@ -263,6 +322,7 @@ namespace astc_helpers { dequant_table m_weights[TOTAL_WEIGHT_ISE_RANGES]; dequant_table m_endpoints[TOTAL_ENDPOINT_ISE_RANGES]; + bool m_initialized_flag = false; const dequant_table& get_weight_tab(uint32_t range) const { @@ -288,16 +348,19 @@ namespace astc_helpers return m_endpoints[range - FIRST_VALID_ENDPOINT_ISE_RANGE]; } - void init(bool init_rank_tabs) + void init() { + if (m_initialized_flag) + return; + for (uint32_t range = FIRST_VALID_WEIGHT_ISE_RANGE; range <= LAST_VALID_WEIGHT_ISE_RANGE; range++) { const uint32_t num_levels = get_ise_levels(range); dequant_table& tab = get_weight_tab(range); - tab.init(true, num_levels, init_rank_tabs); + tab.init(true, num_levels); - create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), init_rank_tabs ? tab.m_ISE_to_rank.data() : nullptr, init_rank_tabs ? tab.m_rank_to_ISE.data() : nullptr, range, true); + create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), tab.m_ISE_to_rank.data(), tab.m_rank_to_ISE.data(), range, true); } for (uint32_t range = FIRST_VALID_ENDPOINT_ISE_RANGE; range <= LAST_VALID_ENDPOINT_ISE_RANGE; range++) @@ -305,15 +368,17 @@ namespace astc_helpers const uint32_t num_levels = get_ise_levels(range); dequant_table& tab = get_endpoint_tab(range); - tab.init(false, num_levels, init_rank_tabs); + tab.init(false, num_levels); - create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), init_rank_tabs ? tab.m_ISE_to_rank.data() : nullptr, init_rank_tabs ? tab.m_rank_to_ISE.data() : nullptr, range, false); + create_quant_tables(tab.m_val_to_ise.data(), tab.m_ISE_to_val.data(), tab.m_ISE_to_rank.data(), tab.m_rank_to_ISE.data(), range, false); } + + m_initialized_flag = true; } }; extern dequant_tables g_dequant_tables; - void init_tables(bool init_rank_tabs); + void init_tables(); struct weighted_sample { @@ -333,9 +398,22 @@ namespace astc_helpers const uint8_t* pSrc_weights, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] uint8_t* pDst_weights); // [by][bx] - // Procedurally returns the texel partition/subset index given the block coordinate and config. + void upsample_weight_grid_xuastc_ldr( + uint32_t bx, uint32_t by, // destination/to dimension + uint32_t wx, uint32_t wy, // source/from dimension + const uint8_t* pSrc_weights0, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights0, // [by][bx] + const uint8_t* pSrc_weights1, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights1); // [by][bx] + + bool is_small_block(uint32_t block_width, uint32_t block_height); + + // Procedurally returns the texel partition/subset index given the block coordinate and config (very slow). int compute_texel_partition(uint32_t seedIn, uint32_t xIn, uint32_t yIn, uint32_t zIn, int num_partitions, bool small_block); + // Returns the texel partition/subset index given the block coordinate and config - table lookup, but currently ONLY 2-3 SUBSETS to save RAM. + int get_precomputed_texel_partition(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions); + void blue_contract( int r, int g, int b, int a, int& dr, int& dg, int& db, int& da); @@ -372,7 +450,7 @@ namespace astc_helpers const int MAX_RGB9E5 = 0xff80; void unpack_rgb9e5(uint32_t packed, float& r, float& g, float& b); uint32_t pack_rgb9e5(float r, float g, float b); - + enum decode_mode { cDecodeModeSRGB8 = 0, // returns uint8_t's, not valid on HDR blocks @@ -385,11 +463,71 @@ namespace astc_helpers // pPixels must point to either 32-bit pixel values (SRGB8/LDR8/9E5) or 64-bit pixel values (HDR16) bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode); + // Assuming the ASTC logical block is valid, this checks for the extra XUASTC LDR constraints. + bool is_block_xuastc_ldr(const log_astc_block& log_blk); + + // XUASTC LDR only - primary assumption is the logical block comes directly from our supercompressor. DO NOT call on general ASTC blocks. + bool decode_block_xuastc_ldr(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode, const uint8_t* pUpsampled_weights_to_use = nullptr, uint32_t start_x = 0, uint32_t start_y = 0, uint32_t end_x = 0, uint32_t end_y = 0); + void decode_bise(uint32_t ise_range, uint8_t* pVals, uint32_t num_vals, const uint8_t *pBits128, uint32_t bit_ofs); // Unpack a physical ASTC encoded GPU texture block to a logical block description. bool unpack_block(const void* pASTC_block, log_astc_block& log_blk, uint32_t blk_width, uint32_t blk_height); + uint8_t& get_weight(log_astc_block& log_block, uint32_t plane_index, uint32_t idx); + uint8_t get_weight(const log_astc_block& log_block, uint32_t plane_index, uint32_t idx); + void extract_weights(const log_astc_block& log_block, uint8_t* pWeights, uint32_t plane_index); + void set_weights(log_astc_block& log_block, const uint8_t* pWeights, uint32_t plane_index); + uint32_t get_total_weights(const log_astc_block& log_block); + + uint8_t* get_endpoints(log_astc_block& log_block, uint32_t partition_index); + const uint8_t* get_endpoints(const log_astc_block& log_block, uint32_t partition_index); + + const char* get_cem_name(uint32_t cem_index); + bool cem_is_ldr_direct(uint32_t cem_index); + bool cem_is_ldr_base_scale(uint32_t cem_index); + bool cem_is_ldr_base_plus_ofs(uint32_t cem_index); + + bool cem_supports_bc(uint32_t cem); + + void bit_transfer_signed_dec(int& a, int& b); + void bit_transfer_signed_enc(int& a, int& b); + + bool cem8_or_12_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + bool cem9_or_13_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + bool used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index); + + uint32_t get_base_cem_without_alpha(uint32_t cem); + + int apply_delta_to_bise_endpoint_val(uint32_t endpoint_ise_range, int ise_val, int delta); + + // index range: [0,NUM_ASTC_BLOCK_SIZES-1] + void get_astc_block_size_by_index(uint32_t index, uint32_t& width, uint32_t& height); + + // -1 if invalid + int find_astc_block_size_index(uint32_t width, uint32_t height); + + // 8-bit linear8 or sRGB8, le/he are [0,255], w is [0,64] + inline int channel_interpolate(int le, int he, int w, bool astc_srgb_decode) + { + assert((w >= 0) && (w <= 64)); + assert((le >= 0) && (le <= 255)); + assert((he >= 0) && (he <= 255)); + + if (astc_srgb_decode) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + return astc_helpers::weight_interpolate(le, he, w) >> 8; + } + } // namespace astc_helpers #endif // BASISU_ASTC_HELPERS_HEADER @@ -403,11 +541,11 @@ namespace astc_helpers template inline T my_min(T a, T b) { return (a < b) ? a : b; } template inline T my_max(T a, T b) { return (a > b) ? a : b; } - const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2] = { - { 4, 4 }, { 5, 4 }, { 5, 5 }, { 6, 5 }, - { 6, 6 }, { 8, 5 }, { 8, 6 }, { 10, 5 }, - { 10, 6 }, { 8, 8 }, { 10, 8 }, { 10, 10 }, - { 12, 10 }, { 12, 12 } + const uint8_t g_astc_block_sizes[NUM_ASTC_BLOCK_SIZES][2] = { + { 4, 4 }, { 5, 4 }, { 5, 5 }, { 6, 5 }, + { 6, 6 }, { 8, 5 }, { 8, 6 }, { 10, 5 }, + { 10, 6 }, { 8, 8 }, { 10, 8 }, { 10, 10 }, + { 12, 10 }, { 12, 12 } }; const int8_t g_ise_range_table[TOTAL_ISE_RANGES][3] = @@ -436,7 +574,7 @@ namespace astc_helpers { 6, 1, 0 }, // 0..191 19 { 8, 0, 0 }, // 0..255 20 }; - + static inline void astc_set_bits_1_to_9(uint32_t* pDst, uint32_t& bit_offset, uint32_t code, uint32_t codesize) { uint8_t* pBuf = reinterpret_cast(pDst); @@ -551,10 +689,10 @@ namespace astc_helpers // Now interleave the 8 encoded trit bits with the bits to form the encoded output. See table 94. astc_set_bits(pOutput, bit_pos, bits[0] | (astc_extract_bits(T, 0, 1) << n) | (bits[1] << (2 + n)), n * 2 + 2); - + astc_set_bits(pOutput, bit_pos, astc_extract_bits(T, 2, 3) | (bits[2] << 2) | (astc_extract_bits(T, 4, 4) << (2 + n)) | (bits[3] << (3 + n)) | (astc_extract_bits(T, 5, 6) << (3 + n * 2)) | (bits[4] << (5 + n * 2)) | (astc_extract_bits(T, 7, 7) << (5 + n * 3)), n * 3 + 6); - + if (pStats) *pStats += n * 5 + 8; } @@ -582,7 +720,7 @@ namespace astc_helpers if (group_size) { - // Range has trits or quints - pack each group of 5 or 3 values + // Range has trits or quints - pack each group of 5 or 3 values const int total_groups = (group_size == 5) ? ((num_vals + 4) / 5) : ((num_vals + 2) / 3); for (int group_index = 0; group_index < total_groups; group_index++) @@ -593,7 +731,7 @@ namespace astc_helpers for (int i = 0; i < limit; i++) vals[i] = pSrc_vals[group_index * group_size + i]; - // Note this always writes a group of 3 or 5 bits values, even for incomplete groups. So it can write more than needed. + // Note this always writes a group of 3 or 5 bits values, even for incomplete groups. So it can write more than needed. // get_ise_sequence_bits() returns the # of bits that must be written for proper decoding. if (group_size == 5) astc_encode_trits(temp, vals, bit_pos, num_bits, pStats); @@ -632,14 +770,14 @@ namespace astc_helpers const uint32_t P = log_block.m_weight_ise_range >= 6; // high precision const uint32_t Dp_P = (log_block.m_dual_plane << 1) | P; // pack dual plane+high precision bits - + // See Tables 81-82 // Compute p from weight range uint32_t p = 2 + log_block.m_weight_ise_range - (P ? 6 : 0); - + // Rearrange p's bits to p0 p2 p1 p = (p >> 1) + ((p & 1) << 2); - + // Try encoding each row of table 82. // W+4 H+2 @@ -676,7 +814,7 @@ namespace astc_helpers config_bits = (Dp_P << 9) | ((W) << 7) | ((H - 2) << 5) | ((p & 4) << 2) | 12 | (p & 3); return true; } - + // 12 H+2 if ((W == 12) && is_packable(H - 2, 2)) { @@ -704,7 +842,7 @@ namespace astc_helpers config_bits = (Dp_P << 9) | (0b1101 << 5) | (p << 2); return true; } - + // W+6 H+6 (no dual plane or high prec) if ((!Dp_P) && is_packable(W - 6, 2) && is_packable(H - 6, 2)) { @@ -715,9 +853,19 @@ namespace astc_helpers // Failed: unsupported weight grid dimensions or config. return false; } - - bool pack_astc_block(astc_block& phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range, pack_stats *pStats) + + bool pack_astc_block(astc_block& phys_block, const log_astc_block& log_block, int* pExpected_endpoint_range, pack_stats *pStats, uint32_t validate_flags) { + // Basic sanity checking + if (!log_block.m_dual_plane) + { + assert(log_block.m_color_component_selector == 0); + } + else + { + assert(log_block.m_color_component_selector <= 3); + } + memset(&phys_block, 0, sizeof(phys_block)); if (pExpected_endpoint_range) @@ -726,7 +874,7 @@ namespace astc_helpers assert(!log_block.m_error_flag); if (log_block.m_error_flag) return false; - + if (log_block.m_solid_color_flag_ldr) { pack_void_extent_ldr(phys_block, log_block.m_solid_color[0], log_block.m_solid_color[1], log_block.m_solid_color[2], log_block.m_solid_color[3], pStats); @@ -737,7 +885,7 @@ namespace astc_helpers pack_void_extent_hdr(phys_block, log_block.m_solid_color[0], log_block.m_solid_color[1], log_block.m_solid_color[2], log_block.m_solid_color[3], pStats); return true; } - + if ((log_block.m_num_partitions < 1) || (log_block.m_num_partitions > MAX_PARTITIONS)) return false; @@ -753,7 +901,7 @@ namespace astc_helpers return false; // TODO: sanity check grid width/height vs. block's physical width/height - + uint32_t config_bits = 0; if (!get_config_bits(log_block, config_bits)) return false; @@ -794,7 +942,7 @@ namespace astc_helpers if (highest_cem > 15) return false; - + // Ensure CEM range is contiguous if (((highest_cem >> 2) > (1 + (lowest_cem >> 2)))) return false; @@ -809,7 +957,7 @@ namespace astc_helpers for (uint32_t j = 0; j < log_block.m_num_partitions; j++) { const int M = log_block.m_color_endpoint_modes[j] & 3; - + const int C = (log_block.m_color_endpoint_modes[j] >> 2) - ((encoded_cem & 3) - 1); if ((C & 1) != C) return false; @@ -850,7 +998,7 @@ namespace astc_helpers return false; total_extra_bits += 2; - + uint32_t ccs_bit_pos = 128 - (int)total_weight_bits - (int)total_extra_bits; astc_set_bits(&phys_block.m_vals[0], ccs_bit_pos, log_block.m_color_component_selector, 2); if (pStats) @@ -868,6 +1016,9 @@ namespace astc_helpers if (total_cem_vals > MAX_ENDPOINTS) return false; + + if (validate_flags & cValidateEarlyOutAtEndpointISEChecks) + return true; int endpoint_ise_range = -1; for (int k = 20; k > 0; k--) @@ -898,6 +1049,9 @@ namespace astc_helpers pStats->m_weight_bits += get_ise_sequence_bits(total_grid_weights, log_block.m_weight_ise_range); } + if (validate_flags & cValidateSkipFinalEndpointWeightPacking) + return true; + // Pack endpoints forwards encode_bise(&phys_block.m_vals[0], log_block.m_endpoints, bit_pos, total_cem_vals, endpoint_ise_range); @@ -1094,12 +1248,12 @@ namespace astc_helpers uint32_t u = 0; switch (ise_range) { - case 0: + case 0: { u = val ? 63 : 0; break; } - case 1: // 0-2 + case 1: // 0-2 { const uint8_t s_tab_0_2[3] = { 0, 32, 63 }; u = s_tab_0_2[val]; @@ -1140,7 +1294,7 @@ namespace astc_helpers const uint32_t num_bits = g_ise_range_table[ise_range][0]; const uint32_t num_trits = g_ise_range_table[ise_range][1]; BASISU_NOTE_UNUSED(num_trits); const uint32_t num_quints = g_ise_range_table[ise_range][2]; BASISU_NOTE_UNUSED(num_quints); - + // compute Table 103 row index const int range_index = num_bits * 2 + (num_quints ? 1 : 0); @@ -1153,11 +1307,11 @@ namespace astc_helpers // Now dequantize // See Table 103. ASTC weight unquantization parameters static const uint32_t C_table[5] = { 50, 28, 23, 13, 11 }; - + const uint32_t a = bits & 1, b = (bits >> 1) & 1, c = (bits >> 2) & 1; const uint32_t A = (a == 0) ? 0 : 0x7F; - + uint32_t B = 0; if (range_index == 4) B = ((b << 6) | (b << 2) | (b << 0)); @@ -1210,7 +1364,7 @@ namespace astc_helpers uint32_t find_nearest_bise_weight(int v, uint32_t ise_range) { assert(ise_range >= FIRST_VALID_WEIGHT_ISE_RANGE && ise_range <= LAST_VALID_WEIGHT_ISE_RANGE); - assert(v <= (int)MAX_WEIGHT_VALUE); + assert(v <= (int)MAX_WEIGHT_INTERPOLANT_VALUE); const uint32_t total_levels = get_ise_levels(ise_range); int best_e = INT_MAX, best_index = 0; @@ -1237,7 +1391,7 @@ namespace astc_helpers uint32_t ise_range, // ise range, [4,20] for endpoints, [0,11] for weights bool weight_flag) // false if block endpoints, true if weights { - const uint32_t num_dequant_vals = weight_flag ? (MAX_WEIGHT_VALUE + 1) : 256; + const uint32_t num_dequant_vals = weight_flag ? (MAX_WEIGHT_INTERPOLANT_VALUE + 1) : 256; for (uint32_t i = 0; i < num_dequant_vals; i++) { @@ -1273,22 +1427,22 @@ namespace astc_helpers for (uint32_t i = 0; i < num_levels; i++) { uint32_t v = weight_flag ? astc_helpers::dequant_bise_weight(i, ise_range) : astc_helpers::dequant_bise_endpoint(i, ise_range); - + // Low=ISE value // High=dequantized value vals[i] = (v << 16) | i; } - + // Sorts by dequantized value std::sort(vals, vals + num_levels); - + for (uint32_t rank = 0; rank < num_levels; rank++) { uint32_t ise_val = (uint8_t)vals[rank]; if (pISE_to_rank) pISE_to_rank[ise_val] = (uint8_t)rank; - + if (pRank_to_ISE) pRank_to_ISE[rank] = (uint8_t)ise_val; } @@ -1314,17 +1468,17 @@ namespace astc_helpers pDst[15] = (uint8_t)(ah >> 8); if (pStats) - pStats->m_header_bits += 128; + pStats->m_header_bits += 16 + 64; } // rh-ah are half-floats - void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats *pStats) + void pack_void_extent_hdr(astc_block& blk, uint16_t rh, uint16_t gh, uint16_t bh, uint16_t ah, pack_stats *pStats) { uint8_t* pDst = (uint8_t*)&blk.m_vals[0]; memset(pDst, 0xFF, 16); pDst[0] = 0b11111100; - + pDst[8] = (uint8_t)rh; pDst[9] = (uint8_t)(rh >> 8); pDst[10] = (uint8_t)gh; @@ -1335,9 +1489,9 @@ namespace astc_helpers pDst[15] = (uint8_t)(ah >> 8); if (pStats) - pStats->m_header_bits += 128; + pStats->m_header_bits += 8 + 64; } - + bool is_cem_ldr(uint32_t mode) { switch (mode) @@ -1356,39 +1510,132 @@ namespace astc_helpers default: break; } + + return false; + } + + bool does_cem_have_alpha(uint32_t mode) + { + switch (mode) + { + case CEM_LDR_LUM_ALPHA_DIRECT: + case CEM_LDR_LUM_ALPHA_BASE_PLUS_OFS: + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + case CEM_HDR_RGB_LDR_ALPHA: + case CEM_HDR_RGB_HDR_ALPHA: + return true; + default: + break; + } return false; } bool is_valid_block_size(uint32_t w, uint32_t h) { - assert((w >= MIN_BLOCK_DIM) && (w <= MAX_BLOCK_DIM)); - assert((h >= MIN_BLOCK_DIM) && (h <= MAX_BLOCK_DIM)); +#define BU_ASTC_HELPERS_SIZECHK(x, y) if ((w == (x)) && (h == (y))) return true; + BU_ASTC_HELPERS_SIZECHK(4, 4); // 0 + BU_ASTC_HELPERS_SIZECHK(5, 4); // 1 -#define SIZECHK(x, y) if ((w == (x)) && (h == (y))) return true; - SIZECHK(4, 4); - SIZECHK(5, 4); + BU_ASTC_HELPERS_SIZECHK(5, 5); // 2 - SIZECHK(5, 5); + BU_ASTC_HELPERS_SIZECHK(6, 5); // 3 + BU_ASTC_HELPERS_SIZECHK(6, 6); // 4 - SIZECHK(6, 5); - SIZECHK(6, 6); + BU_ASTC_HELPERS_SIZECHK(8, 5); // 5 + BU_ASTC_HELPERS_SIZECHK(8, 6); // 6 + BU_ASTC_HELPERS_SIZECHK(10, 5); // 7 + BU_ASTC_HELPERS_SIZECHK(10, 6); // 8 - SIZECHK(8, 5); - SIZECHK(8, 6); - SIZECHK(10, 5); - SIZECHK(10, 6); + BU_ASTC_HELPERS_SIZECHK(8, 8); // 9 + BU_ASTC_HELPERS_SIZECHK(10, 8); // 10 + BU_ASTC_HELPERS_SIZECHK(10, 10); // 11 - SIZECHK(8, 8); - SIZECHK(10, 8); - SIZECHK(10, 10); - - SIZECHK(12, 10); - SIZECHK(12, 12); -#undef SIZECHK + BU_ASTC_HELPERS_SIZECHK(12, 10); // 12 + BU_ASTC_HELPERS_SIZECHK(12, 12); // 13 +#undef BU_ASTC_HELPERS_SIZECHK return false; } + + uint32_t get_block_size_index(uint32_t w, uint32_t h) + { + assert(is_valid_block_size(w, h)); + + const uint32_t t = w * h; + + if (t <= 36) + { + if (t == 36) + return cBLOCK_SIZE_6x6; + else if (t == 16) + return cBLOCK_SIZE_4x4; + else if (t == 25) + return cBLOCK_SIZE_5x5; + else if (t == 20) + return cBLOCK_SIZE_5x4; + else if (t == 30) + return cBLOCK_SIZE_6x5; + } + else if (t <= 64) + { + if (t == 64) + return cBLOCK_SIZE_8x8; + else if (t == 60) + return cBLOCK_SIZE_10x6; + else if (t == 50) + return cBLOCK_SIZE_10x5; + else if (t == 48) + return cBLOCK_SIZE_8x6; + else if (t == 40) + return cBLOCK_SIZE_8x5; + } + else + { + if (t == 80) + return cBLOCK_SIZE_10x8; + else if (t == 100) + return cBLOCK_SIZE_10x10; + else if (t == 120) + return cBLOCK_SIZE_12x10; + else if (t == 144) + return cBLOCK_SIZE_12x12; + } + + assert(0); + return cBLOCK_SIZE_4x4; + } + + // returns the standard ASTC bitrates given a valid block size from the ASTC spec. + // 0=invalid block size + float get_bitrate_from_block_size(uint32_t w, uint32_t h) + { +#define BU_ASTC_HELPERS_BLOCK_BITRATE(x, y, b) if ((w == (x)) && (h == (y))) return (b); + BU_ASTC_HELPERS_BLOCK_BITRATE(4, 4, 8.0f); + BU_ASTC_HELPERS_BLOCK_BITRATE(5, 4, 6.4f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(5, 5, 5.12f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(6, 5, 4.27f); + BU_ASTC_HELPERS_BLOCK_BITRATE(6, 6, 3.56f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 5, 3.20f); + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 6, 2.67f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 5, 2.56f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 6, 2.13f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(8, 8, 2.00f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 8, 1.60f); + BU_ASTC_HELPERS_BLOCK_BITRATE(10, 10, 1.28f); + + BU_ASTC_HELPERS_BLOCK_BITRATE(12, 10, 1.07f); + BU_ASTC_HELPERS_BLOCK_BITRATE(12, 12, .89f); +#undef BU_ASTC_HELPERS_BLOCK_BITRATE + + return 0.0f; + } bool block_has_any_hdr_cems(const log_astc_block& log_blk) { @@ -1411,20 +1658,19 @@ namespace astc_helpers return false; } - + dequant_tables g_dequant_tables; - - void precompute_texel_partitions_4x4(); - void precompute_texel_partitions_6x6(); - - void init_tables(bool init_rank_tabs) + + void precompute_texel_partitions(); + + // TODO: this is called twice when using the encoder, first init_rank_tabs=false then init_rank_tabs=true. + void init_tables() { - g_dequant_tables.init(init_rank_tabs); - - precompute_texel_partitions_4x4(); - precompute_texel_partitions_6x6(); + g_dequant_tables.init(); + + precompute_texel_partitions(); } - + void compute_upsample_weights( int block_width, int block_height, int weight_grid_width, int weight_grid_height, @@ -1474,6 +1720,8 @@ namespace astc_helpers if (total_src_weights == total_dst_weights) { + assert((bx == wx) && (by == wy)); + memcpy(pDst_weights, pSrc_weights, total_src_weights); return; } @@ -1507,6 +1755,78 @@ namespace astc_helpers } } + void upsample_weight_grid_xuastc_ldr( + uint32_t bx, uint32_t by, // destination/to dimension + uint32_t wx, uint32_t wy, // source/from dimension + const uint8_t* pSrc_weights0, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights0, // [by][bx] + const uint8_t* pSrc_weights1, // these are dequantized [0,64] weights, NOT ISE symbols, [wy][wx] + uint8_t* pDst_weights1) // [by][bx] + { + assert((bx >= 2) && (by >= 2) && (bx <= 12) && (by <= 12)); + assert((wx >= 2) && (wy >= 2) && (wx <= bx) && (wy <= by)); + + assert((bx != wx) || (by != wy)); + + const uint32_t scaleX = (1024 + bx / 2) / (bx - 1); + const uint32_t scaleY = (1024 + by / 2) / (by - 1); + + const uint32_t gYUInc = scaleY * (wy - 1); + const uint32_t gXUInc = scaleX * (wx - 1); + + uint32_t gYU = 32; + for (uint32_t texel_y = 0; texel_y < by; texel_y++) + { + const uint32_t gY = gYU >> 6; + gYU += gYUInc; + + const uint32_t jY = gY >> 4; + const uint32_t fY = gY & 0xf; + + uint32_t gXU = 32; + for (uint32_t texel_x = 0; texel_x < bx; texel_x++) + { + const uint32_t gX = gXU >> 6; + gXU += gXUInc; + + const uint32_t jX = gX >> 4; + const uint32_t fX = gX & 0xf; + + const uint32_t w11 = (fX * fY + 8) >> 4; + const uint32_t w10 = fY - w11; + const uint32_t w01 = fX - w11; + const uint32_t w00 = 16 - fX - fY + w11; + + assert(w00 || w01 || w10 || w11); + + const uint32_t sx = jX, sy = jY; + + { + uint32_t total0 = 8; + + if (w00) total0 += pSrc_weights0[sx + sy * wx] * w00; + if (w01) total0 += pSrc_weights0[sx + 1 + sy * wx] * w01; + if (w10) total0 += pSrc_weights0[sx + (sy + 1) * wx] * w10; + if (w11) total0 += pSrc_weights0[sx + 1 + (sy + 1) * wx] * w11; + + pDst_weights0[texel_x + texel_y * bx] = (uint8_t)(total0 >> 4); + } + + if (pDst_weights1) + { + uint32_t total1 = 8; + + if (w00) total1 += pSrc_weights1[sx + sy * wx] * w00; + if (w01) total1 += pSrc_weights1[sx + 1 + sy * wx] * w01; + if (w10) total1 += pSrc_weights1[sx + (sy + 1) * wx] * w10; + if (w11) total1 += pSrc_weights1[sx + 1 + (sy + 1) * wx] * w11; + + pDst_weights1[texel_x + texel_y * bx] = (uint8_t)(total1 >> 4); + } + } // texel_x + } // texel_y + } + inline uint32_t hash52(uint32_t v) { uint32_t p = v; @@ -1516,6 +1836,16 @@ namespace astc_helpers return p; } + bool is_small_block(uint32_t block_width, uint32_t block_height) + { + assert((block_width >= MIN_BLOCK_DIM) && (block_width <= MAX_BLOCK_DIM)); + assert((block_height >= MIN_BLOCK_DIM) && (block_height <= MAX_BLOCK_DIM)); + + const uint32_t num_blk_pixels = block_width * block_height; + + return num_blk_pixels < 31; + } + // small_block = num_blk_pixels < 31 int compute_texel_partition(uint32_t seedIn, uint32_t xIn, uint32_t yIn, uint32_t zIn, int num_partitions, bool small_block) { @@ -1582,76 +1912,108 @@ namespace astc_helpers : (c >= d) ? 2 : 3; } + + // Precomputed partition patterns for each 10-bit seed and small/large block sizes for 2-3 subsets. + // This costs 144KB of RAM and some init, but considering the sheer complexity of compute_texel_partition() and how hotly it's called in the compressors and transcoders that's worth it. + // Byte packing: + // low 4 bits=small blocks (on valid up to 6x5) + // high 4 bits=large blocks (6x6 or larger) - // 4x4, 2 and 3 subsets - static uint32_t g_texel_partitions_4x4[1024][2]; + static uint8_t g_texel_partitions[NUM_PARTITION_PATTERNS][12][12]; // [seed][y][x] - // 6x6, 2 and 3 subsets (2 subsets low 4 bits, 3 subsets high 4 bits) - static uint8_t g_texel_partitions_6x6[1024][6 * 6]; - - void precompute_texel_partitions_4x4() + void sanity_check_texel_partition_tables() { - for (uint32_t p = 0; p < 1024; p++) +#if 0 +#if defined(_DEBUG) || defined(DEBUG) + // sanity checking + for (uint32_t i = 0; i < cTOTAL_BLOCK_SIZES; i++) { - uint32_t v2 = 0, v3 = 0; + const uint32_t bw = g_astc_block_sizes[i][0], bh = g_astc_block_sizes[i][1]; + const bool is_small_block_flag = is_small_block(bw, bh); - for (uint32_t y = 0; y < 4; y++) + assert(get_block_size_index(bw, bh) == i); + + for (uint32_t s = 0; s < NUM_PARTITION_PATTERNS; s++) { - for (uint32_t x = 0; x < 4; x++) + for (uint32_t y = 0; y < bh; y++) { - const uint32_t shift = x * 2 + y * 8; - v2 |= (compute_texel_partition(p, x, y, 0, 2, true) << shift); - v3 |= (compute_texel_partition(p, x, y, 0, 3, true) << shift); - } - } + for (uint32_t x = 0; x < bw; x++) + { + const uint32_t k2 = compute_texel_partition(s, x, y, 0, 2, is_small_block_flag); + const uint32_t k3 = compute_texel_partition(s, x, y, 0, 3, is_small_block_flag); - g_texel_partitions_4x4[p][0] = v2; - g_texel_partitions_4x4[p][1] = v3; + assert(get_precomputed_texel_partition(bw, bh, s, x, y, 2) == (int)k2); + assert(get_precomputed_texel_partition(bw, bh, s, x, y, 3) == (int)k3); + } // x + } // y + } // s } + printf("precompute_texel_partitions: Sanity check OK\n"); +#endif +#endif } - - void precompute_texel_partitions_6x6() + + void precompute_texel_partition() { - for (uint32_t p = 0; p < 1024; p++) + for (uint32_t seed = 0; seed < NUM_PARTITION_PATTERNS; seed++) { - for (uint32_t y = 0; y < 6; y++) + for (uint32_t y = 0; y < MAX_BLOCK_DIM; y++) { - for (uint32_t x = 0; x < 6; x++) + for (uint32_t x = 0; x < MAX_BLOCK_DIM; x++) { - const uint32_t p2 = compute_texel_partition(p, x, y, 0, 2, false); - const uint32_t p3 = compute_texel_partition(p, x, y, 0, 3, false); + uint32_t k = 0; - assert((p2 <= 1) && (p3 <= 2)); - g_texel_partitions_6x6[p][x + y * 6] = (uint8_t)((p3 << 4) | p2); - } - } - } + // small block (width*height<31) + if ((x <= 6) && (y <= 5)) + { + uint32_t v2 = compute_texel_partition(seed, x, y, 0, 2, true); assert(v2 <= 1); + uint32_t v3 = compute_texel_partition(seed, x, y, 0, 3, true); assert(v3 <= 2); + k |= v2 | (v3 << 2); + } + + // not small block + { + uint32_t v2 = compute_texel_partition(seed, x, y, 0, 2, false); assert(v2 <= 1); + uint32_t v3 = compute_texel_partition(seed, x, y, 0, 3, false); assert(v3 <= 2); + k |= ((v2 | (v3 << 2)) << 4); + } + + assert(k <= 255); + + g_texel_partitions[seed][y][x] = (uint8_t)k; + } // x + } // y + } // seed } - - static inline int get_precompute_texel_partitions_4x4(uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions) + + int get_precomputed_texel_partition(uint32_t block_width, uint32_t block_height, uint32_t seed, uint32_t x, uint32_t y, uint32_t subsets) { - assert(g_texel_partitions_4x4[1][0]); - assert(seed < 1024); - assert((x <= 3) && (y <= 3)); - assert((num_partitions >= 2) && (num_partitions <= 3)); + assert(seed < NUM_PARTITION_PATTERNS); + assert((subsets >= 2) && (subsets <= 3)); + assert((x < block_width) && (y < block_height)); + + const uint32_t v = g_texel_partitions[seed][y][x]; - const uint32_t shift = x * 2 + y * 8; - return (g_texel_partitions_4x4[seed][num_partitions - 2] >> shift) & 3; + uint32_t shift = (subsets == 3) ? 2 : 0; + shift += ((block_width * block_height) >= 31) * 4; + uint32_t res = (v >> shift) & 3; + + // sanity checking + assert(res == (uint32_t)compute_texel_partition(seed, x, y, 0, subsets, is_small_block(block_width, block_height))); + + return res; } - - static inline int get_precompute_texel_partitions_6x6(uint32_t seed, uint32_t x, uint32_t y, uint32_t num_partitions) + + void precompute_texel_partitions() { - assert(g_texel_partitions_6x6[0][0]); - assert(seed < 1024); - assert((x <= 5) && (y <= 5)); - assert((num_partitions >= 2) && (num_partitions <= 3)); + if (!g_texel_partitions[0][0][0]) + precompute_texel_partition(); - const uint32_t shift = (num_partitions == 3) ? 4 : 0; - return (g_texel_partitions_6x6[seed][x + y * 6] >> shift) & 3; + sanity_check_texel_partition_tables(); } void blue_contract( - int r, int g, int b, int a, + int r, int g, int b, int a, int &dr, int &dg, int &db, int &da) { dr = (r + b) >> 1; @@ -1666,7 +2028,7 @@ namespace astc_helpers b |= (a & 0x80); a >>= 1; a &= 0x3F; - if ((a & 0x20) != 0) + if ((a & 0x20) != 0) a -= 0x40; } @@ -1900,7 +2262,7 @@ namespace astc_helpers e0_g = y0; e1_g = y1; e0_b = y0; e1_b = y1; e0_a = 0x780; e1_a = 0x780; - + break; } case CEM_HDR_LUM_SMALL_RANGE: @@ -1917,11 +2279,11 @@ namespace astc_helpers y0 = ((v1 & 0xF0) << 4) | ((v0 & 0x7F) << 1); d = (v1 & 0x0F) << 1; } - + y1 = y0 + d; - if (y1 > 0xFFF) + if (y1 > 0xFFF) y1 = 0xFFF; - + e0_r = y0; e1_r = y1; e0_g = y0; e1_g = y1; e0_b = y0; e1_b = y1; @@ -1932,36 +2294,36 @@ namespace astc_helpers case CEM_HDR_RGB_BASE_SCALE: { int v2 = pE[2], v3 = pE[3]; - + int modeval = ((v0 & 0xC0) >> 6) | ((v1 & 0x80) >> 5) | ((v2 & 0x80) >> 4); - + int majcomp, mode; - if ((modeval & 0xC) != 0xC) + if ((modeval & 0xC) != 0xC) { - majcomp = modeval >> 2; + majcomp = modeval >> 2; mode = modeval & 3; } - else if (modeval != 0xF) + else if (modeval != 0xF) { - majcomp = modeval & 3; + majcomp = modeval & 3; mode = 4; } - else + else { - majcomp = 0; + majcomp = 0; mode = 5; } - int red = v0 & 0x3f; + int red = v0 & 0x3f; int green = v1 & 0x1f; - int blue = v2 & 0x1f; + int blue = v2 & 0x1f; int scale = v3 & 0x1f; - int x0 = (v1 >> 6) & 1; - int x1 = (v1 >> 5) & 1; + int x0 = (v1 >> 6) & 1; + int x1 = (v1 >> 5) & 1; int x2 = (v2 >> 6) & 1; - int x3 = (v2 >> 5) & 1; - int x4 = (v3 >> 7) & 1; + int x3 = (v2 >> 5) & 1; + int x4 = (v3 >> 7) & 1; int x5 = (v3 >> 6) & 1; int x6 = (v3 >> 5) & 1; @@ -1985,25 +2347,25 @@ namespace astc_helpers if (ohm & 0x02) red |= x5 << 10; static const int s_shamts[6] = { 1,1,2,3,4,5 }; - + const int shamt = s_shamts[mode]; - red <<= shamt; - green <<= shamt; - blue <<= shamt; + red <<= shamt; + green <<= shamt; + blue <<= shamt; scale <<= shamt; - if (mode != 5) - { - green = red - green; - blue = red - blue; + if (mode != 5) + { + green = red - green; + blue = red - blue; } - if (majcomp == 1) + if (majcomp == 1) std::swap(red, green); - if (majcomp == 2) + if (majcomp == 2) std::swap(red, blue); - + e1_r = clamp(red, 0, 0xFFF); e1_g = clamp(green, 0, 0xFFF); e1_b = clamp(blue, 0, 0xFFF); @@ -2027,7 +2389,7 @@ namespace astc_helpers e0_a = 0x780; e1_a = 0x780; - if (majcomp == 3) + if (majcomp == 3) { e0_r = v0 << 4; e0_g = v2 << 4; @@ -2128,12 +2490,16 @@ namespace astc_helpers v7 &= (0x3F >> mode); v7 ^= (0x20 >> mode); v7 -= (0x20 >> mode); - v6 <<= (4 - mode); - v7 <<= (4 - mode); + + //v6 <<= (4 - mode); // undefined behavior if neg + v6 = ((uint32_t)v6) << (4 - mode); + + //v7 <<= (4 - mode); // undefined behavior if neg + v7 = ((uint32_t)v7) << (4 - mode); v7 += v6; v7 = clamp(v7, 0, 0xFFF); - e0_a = v6; + e0_a = v6; e1_a = v7; } } @@ -2152,7 +2518,7 @@ namespace astc_helpers } } } - + static inline bool is_half_inf_or_nan(half_float v) { return get_bits(v, 10, 14) == 31; @@ -2265,7 +2631,7 @@ namespace astc_helpers x.u = m | (e << 23) | (s << 31); return x.f; } - + // See https://registry.khronos.org/OpenGL/extensions/EXT/EXT_texture_shared_exponent.txt const int RGB9E5_EXPONENT_BITS = 5, RGB9E5_MANTISSA_BITS = 9, RGB9E5_EXP_BIAS = 15, RGB9E5_MAX_VALID_BIASED_EXP = 31; const int MAX_RGB9E5_EXP = (RGB9E5_MAX_VALID_BIASED_EXP - RGB9E5_EXP_BIAS); @@ -2273,7 +2639,7 @@ namespace astc_helpers const int MAX_RGB9E5_MANTISSA = (RGB9E5_MANTISSA_VALUES - 1); //const int MAX_RGB9E5 = (int)(((float)MAX_RGB9E5_MANTISSA) / RGB9E5_MANTISSA_VALUES * (1 << MAX_RGB9E5_EXP)); const int EPSILON_RGB9E5 = (int)((1.0f / (float)RGB9E5_MANTISSA_VALUES) / (float)(1 << RGB9E5_EXP_BIAS)); - + void unpack_rgb9e5(uint32_t packed, float& r, float& g, float& b) { int x = packed & 511; @@ -2287,9 +2653,9 @@ namespace astc_helpers g = y * scale; b = z * scale; } - + // floor_log2 is not correct for the denorm and zero values, but we are going to do a max of this value with the minimum rgb9e5 exponent that will hide these problem cases. - static inline int floor_log2(float x) + static inline int floor_log2(float x) { union float754 { @@ -2325,7 +2691,7 @@ namespace astc_helpers exp_shared += 1; assert(exp_shared <= RGB9E5_MAX_VALID_BIASED_EXP); } - else + else { assert(maxm <= MAX_RGB9E5_MANTISSA); } @@ -2337,7 +2703,7 @@ namespace astc_helpers assert((rm >= 0) && (rm <= MAX_RGB9E5_MANTISSA)); assert((gm >= 0) && (gm <= MAX_RGB9E5_MANTISSA)); assert((bm >= 0) && (bm <= MAX_RGB9E5_MANTISSA)); - + return rm | (gm << 9) | (bm << 18) | (exp_shared << 27); } @@ -2348,7 +2714,7 @@ namespace astc_helpers if (!x) return 17; - + uint32_t n = 0; while ((x & 0x10000) == 0) { @@ -2426,17 +2792,8 @@ namespace astc_helpers return texel; } - // Important: pPixels is either 32-bit/texel or 64-bit/texel. - bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode) + static void write_error_block(void* pPixels, uint32_t num_blk_pixels, decode_mode dec_mode) { - assert(is_valid_block_size(blk_width, blk_height)); - - assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); - if (!g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()) - return false; - - const uint32_t num_blk_pixels = blk_width * blk_height; - // Write block error color if (dec_mode == cDecodeModeHDR16) { @@ -2455,9 +2812,32 @@ namespace astc_helpers for (uint32_t i = 0; i < num_blk_pixels; i++) ((uint32_t*)pPixels)[i] = 0xFFFF00FF; } + } + + // Important: pPixels is either 32-bit/texel or 64-bit/texel. + bool decode_block(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode) + { + assert(is_valid_block_size(blk_width, blk_height)); + + // Basic sanity checking + if (!log_blk.m_dual_plane) + { + assert(log_blk.m_color_component_selector == 0); + } + else + { + assert(log_blk.m_color_component_selector <= 3); + } + assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); + if (!g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()) + return false; + + const uint32_t num_blk_pixels = blk_width * blk_height; + if (log_blk.m_error_flag) { + write_error_block(pPixels, num_blk_pixels, dec_mode); // Should this return false? It's not an invalid logical block config, though. return false; } @@ -2510,7 +2890,7 @@ namespace astc_helpers float r = half_to_float(log_blk.m_solid_color[0]); float g = half_to_float(log_blk.m_solid_color[1]); float b = half_to_float(log_blk.m_solid_color[2]); - + const uint32_t packed = pack_rgb9e5(r, g, b); for (uint32_t i = 0; i < num_blk_pixels; i++) @@ -2518,36 +2898,71 @@ namespace astc_helpers } else { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; } return true; } - + // Sanity check block's config if ((log_blk.m_grid_width < 2) || (log_blk.m_grid_height < 2)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_grid_width > blk_width) || (log_blk.m_grid_height > blk_height)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } if ((log_blk.m_endpoint_ise_range < FIRST_VALID_ENDPOINT_ISE_RANGE) || (log_blk.m_endpoint_ise_range > LAST_VALID_ENDPOINT_ISE_RANGE)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_weight_ise_range < FIRST_VALID_WEIGHT_ISE_RANGE) || (log_blk.m_weight_ise_range > LAST_VALID_WEIGHT_ISE_RANGE)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_num_partitions < 1) || (log_blk.m_num_partitions > MAX_PARTITIONS)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_dual_plane) && (log_blk.m_num_partitions > MAX_DUAL_PLANE_PARTITIONS)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if (log_blk.m_partition_id >= NUM_PARTITION_PATTERNS) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if ((log_blk.m_num_partitions == 1) && (log_blk.m_partition_id > 0)) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + if (log_blk.m_color_component_selector > 3) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const uint32_t total_endpoint_levels = get_ise_levels(log_blk.m_endpoint_ise_range); const uint32_t total_weight_levels = get_ise_levels(log_blk.m_weight_ise_range); - + bool is_ldr_endpoints[MAX_PARTITIONS]; // Check CEM's @@ -2555,15 +2970,21 @@ namespace astc_helpers for (uint32_t i = 0; i < log_blk.m_num_partitions; i++) { if (log_blk.m_color_endpoint_modes[i] > 15) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } total_cem_vals += get_num_cem_values(log_blk.m_color_endpoint_modes[i]); - + is_ldr_endpoints[i] = is_cem_ldr(log_blk.m_color_endpoint_modes[i]); } if (total_cem_vals > MAX_ENDPOINTS) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const dequant_table& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range); const uint8_t* pEndpoint_dequant = endpoint_dequant_tab.m_ISE_to_val.data(); @@ -2573,21 +2994,28 @@ namespace astc_helpers for (uint32_t i = 0; i < total_cem_vals; i++) { if (log_blk.m_endpoints[i] >= total_endpoint_levels) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } + dequantized_endpoints[i] = pEndpoint_dequant[log_blk.m_endpoints[i]]; } - + // Dequantize weights to [0,64] uint8_t dequantized_weights[2][12 * 12]; - + const dequant_table& weight_dequant_tab = g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); - + const uint32_t total_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; for (uint32_t i = 0; i < total_weight_vals; i++) { if (log_blk.m_weights[i] >= total_weight_levels) + { + write_error_block(pPixels, num_blk_pixels, dec_mode); return false; + } const uint32_t plane_index = log_blk.m_dual_plane ? (i & 1) : 0; const uint32_t grid_index = log_blk.m_dual_plane ? (i >> 1) : i; @@ -2617,10 +3045,9 @@ namespace astc_helpers // Decode texels const bool small_block = num_blk_pixels < 31; - const bool use_precomputed_texel_partitions_4x4 = (blk_width == 4) && (blk_height == 4) && (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); - const bool use_precomputed_texel_partitions_6x6 = (blk_width == 6) && (blk_height == 6) && (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); + const bool use_precomputed_texel_partitions = (log_blk.m_num_partitions >= 2) && (log_blk.m_num_partitions <= 3); const uint32_t ccs = log_blk.m_dual_plane ? log_blk.m_color_component_selector : UINT32_MAX; - + bool success = true; if (dec_mode == cDecodeModeRGB9E5) @@ -2631,14 +3058,15 @@ namespace astc_helpers for (uint32_t x = 0; x < blk_width; x++) { const uint32_t pixel_index = x + y * blk_width; - + uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } @@ -2680,7 +3108,7 @@ namespace astc_helpers if (is_half_inf_or_nan((half_float)comp[c])) comp[c] = 0x7BFF; } - + } // c uint32_t packed; @@ -2697,21 +3125,22 @@ namespace astc_helpers else if (dec_mode == cDecodeModeHDR16) { // Note: must round towards zero when converting float to half for ASTC (18.19 Weight Application) - + // returns half floats for (uint32_t y = 0; y < blk_height; y++) { for (uint32_t x = 0; x < blk_width; x++) { const uint32_t pixel_index = x + y * blk_width; - + uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } @@ -2751,13 +3180,13 @@ namespace astc_helpers int he = endpoints[subset][c][1] << 4; int qlog16 = weight_interpolate(le, he, w); - + o = qlog16_to_half(qlog16); if (is_half_inf_or_nan(o)) o = 0x7BFF; } - + ((half_float*)pPixels)[pixel_index * 4 + c] = o; } @@ -2776,17 +3205,18 @@ namespace astc_helpers uint32_t subset = 0; if (log_blk.m_num_partitions > 1) { - if (use_precomputed_texel_partitions_4x4) - subset = get_precompute_texel_partitions_4x4(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); - else if (use_precomputed_texel_partitions_6x6) - subset = get_precompute_texel_partitions_6x6(log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + if (use_precomputed_texel_partitions) + { + subset = get_precomputed_texel_partition(blk_width, blk_height, log_blk.m_partition_id, x, y, log_blk.m_num_partitions); + //assert((int)subset == compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block)); // extra paranoia + } else subset = compute_texel_partition(log_blk.m_partition_id, x, y, 0, log_blk.m_num_partitions, small_block); } if (!is_ldr_endpoints[subset]) { - ((uint32_t*)pPixels)[pixel_index * 4] = 0xFFFF00FF; + ((uint32_t*)pPixels)[pixel_index] = 0xFFFF00FF; success = false; } else @@ -2801,6 +3231,9 @@ namespace astc_helpers // FIXME: the spec is apparently wrong? this matches ARM's and Google's decoder //if ((dec_mode == cDecodeModeSRGB8) && (c <= 2)) // See https://github.com/ARM-software/astc-encoder/issues/447 + // See latest spec with recent (2023-2024) fixes: + // https://raw.githubusercontent.com/KhronosGroup/DataFormat/refs/heads/main/astc.txt + // "For _LDR endpoint modes_, each color component C is calculated from the corresponding 8 - bit endpoint components C~0~and C~1~as follows" - does this mean alpha too? I guess so. (8/15/2025.) if (dec_mode == cDecodeModeSRGB8) { le = (le << 8) | 0x80; @@ -2814,8 +3247,8 @@ namespace astc_helpers uint32_t k = weight_interpolate(le, he, w); - // FIXME: This is what the spec says to do in LDR mode, but this is not what ARM's decoder does - // See decompress_symbolic_block(), decode_texel() and unorm16_to_sf16. + // FIXME (old comment - before 2023/2024 ARM etc. spec fixes): This is what the spec says to do in LDR mode, but this is not what ARM's decoder does + // See decompress_symbolic_block(), decode_texel() and unorm16_to_sf16. // It seems to effectively divide by 65535.0 and convert to FP16, then back to float, mul by 255.0, add .5 and then convert to 8-bit. ((uint8_t*)pPixels)[pixel_index * 4 + c] = (uint8_t)(k >> 8); } @@ -2824,10 +3257,428 @@ namespace astc_helpers } // x } // y } - + return success; } + bool is_block_xuastc_ldr(const log_astc_block& log_blk) + { + if (log_blk.m_error_flag) + return false; + + if (log_blk.m_solid_color_flag_ldr) + return true; + + if (log_blk.m_solid_color_flag_hdr) + return false; + + if (log_blk.m_num_partitions > 3) + return false; + + if ((log_blk.m_dual_plane) && (log_blk.m_num_partitions > 1)) + return false; + + // TODO: Check partition pattern ID against unique set. + + for (uint32_t i = 1; i < log_blk.m_num_partitions; i++) + if (log_blk.m_color_endpoint_modes[0] != log_blk.m_color_endpoint_modes[i]) + return false; + + switch (log_blk.m_color_endpoint_modes[0]) + { + case CEM_LDR_LUM_DIRECT: + case CEM_LDR_LUM_ALPHA_DIRECT: + case CEM_LDR_RGB_BASE_SCALE: + case CEM_LDR_RGB_DIRECT: + case CEM_LDR_RGB_BASE_PLUS_OFFSET: + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + break; + } + default: + { + return false; + } + } + + return true; + } + + // ~2x faster than decode_block(), but XUASTC LDR only. + // pUpsampled_weights_to_use must be at block res, [0,64], single plane blocks ONLY + bool decode_block_xuastc_ldr(const log_astc_block& log_blk, void* pPixels, uint32_t blk_width, uint32_t blk_height, decode_mode dec_mode, + const uint8_t* pUpsampled_weights_to_use, uint32_t start_x, uint32_t start_y, uint32_t end_x, uint32_t end_y) + { + if (!end_x) + end_x = blk_width; + + if (!end_y) + end_y = blk_height; + + assert(start_x < end_x); + assert(start_y < end_y); + assert(end_x <= blk_width); + assert(end_y <= blk_height); + + assert(g_dequant_tables.m_endpoints[0].m_ISE_to_val.size()); + assert((dec_mode == cDecodeModeSRGB8) || (dec_mode == cDecodeModeLDR8)); + assert(is_valid_block_size(blk_width, blk_height)); + assert(!log_blk.m_error_flag && !log_blk.m_solid_color_flag_hdr); + + if (!log_blk.m_solid_color_flag_ldr) + { + assert(((log_blk.m_num_partitions >= 1) && (log_blk.m_num_partitions <= 3))); + assert((log_blk.m_grid_width >= 2) & (log_blk.m_grid_height >= 2)); + assert((log_blk.m_grid_width <= blk_width) && (log_blk.m_grid_height <= blk_height)); + assert((log_blk.m_grid_width * log_blk.m_grid_height) <= MAX_GRID_WEIGHTS); + assert((log_blk.m_num_partitions > 1) || (log_blk.m_partition_id == 0)); + } + + assert(is_block_xuastc_ldr(log_blk)); + + const uint32_t num_blk_pixels = blk_width * blk_height; + + // Handle solid color blocks + if (log_blk.m_solid_color_flag_ldr) + { + // Convert LDR pixels to 8-bits + uint32_t x; + + ((uint8_t*)&x)[0] = (uint8_t)(log_blk.m_solid_color[0] >> 8); + ((uint8_t*)&x)[1] = (uint8_t)(log_blk.m_solid_color[1] >> 8); + ((uint8_t*)&x)[2] = (uint8_t)(log_blk.m_solid_color[2] >> 8); + ((uint8_t*)&x)[3] = (uint8_t)(log_blk.m_solid_color[3] >> 8); + + uint32_t* pDst = (uint32_t*)pPixels; + + uint32_t i = 0; + while ((i + 3) < num_blk_pixels) + { + pDst[i] = x; + pDst[i + 1] = x; + pDst[i + 2] = x; + pDst[i + 3] = x; + + i += 4; + } + + while (i < num_blk_pixels) + pDst[i++] = x; + + return true; + } + + const dequant_table& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range); + const uint8_t* pEndpoint_dequant = endpoint_dequant_tab.m_ISE_to_val.data(); + + const dequant_table& weight_dequant_tab = g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + // Check CEM's + const uint32_t num_cem_vals = get_num_cem_values(log_blk.m_color_endpoint_modes[0]); + const uint32_t total_cem_vals = num_cem_vals * log_blk.m_num_partitions; + + assert(total_cem_vals <= MAX_ENDPOINTS); + + // Dequantized endpoints to [0,255] + uint8_t dequantized_endpoints[MAX_ENDPOINTS]; + + for (uint32_t i = 0; i < total_cem_vals; i++) + { + assert(log_blk.m_endpoints[i] < endpoint_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_endpoints[i] = pEndpoint_dequant[log_blk.m_endpoints[i]]; + } + + // Decode CEM's + int endpoints[4][4][2]; // [subset][comp][l/h] + + uint32_t endpoint_val_index = 0; + const uint32_t cem_index = log_blk.m_color_endpoint_modes[0]; + + uint32_t alpha_mask = 0xFF; + + for (uint32_t subset = 0; subset < log_blk.m_num_partitions; subset++) + { + assert(log_blk.m_color_endpoint_modes[subset] == cem_index); + + decode_endpoint(cem_index, &endpoints[subset][0], &dequantized_endpoints[endpoint_val_index]); + + alpha_mask &= endpoints[subset][3][0]; + alpha_mask &= endpoints[subset][3][1]; + + endpoint_val_index += num_cem_vals; + } + + const bool any_alpha = alpha_mask != 255; + + // Dequantize weights to [0,64] + uint8_t upsampled_weights[2][12 * 12]; + + const uint32_t total_weight_vals = (log_blk.m_dual_plane ? 2 : 1) * log_blk.m_grid_width * log_blk.m_grid_height; + + // Upsample weight grid. [0,64] weights + const uint8_t(*pUpsampled_weights)[12 * 12]; + + uint8_t dequantized_weights[2][12 * 12]; + + // For simplicity, ignore any passed in weights if dual plane + if ((pUpsampled_weights_to_use) && (!log_blk.m_dual_plane)) + { + // Caller is jamming in already unpacked weights for the first plane to save time + pUpsampled_weights = reinterpret_cast(pUpsampled_weights_to_use); + } + else + { + if (log_blk.m_dual_plane) + { + for (uint32_t i = 0; i < total_weight_vals; i++) + { + const uint32_t plane_index = i & 1; + const uint32_t grid_index = i >> 1; + + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_weights[plane_index][grid_index] = pWeight_dequant[log_blk.m_weights[i]]; + } + } + else + { + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + dequantized_weights[0][i] = pWeight_dequant[log_blk.m_weights[i]]; + } + } + + pUpsampled_weights = &dequantized_weights[0]; + + if ((log_blk.m_grid_width < blk_width) || (log_blk.m_grid_height < blk_height)) + { + upsample_weight_grid_xuastc_ldr(blk_width, blk_height, + log_blk.m_grid_width, log_blk.m_grid_height, + &dequantized_weights[0][0], &upsampled_weights[0][0], + log_blk.m_dual_plane ? &dequantized_weights[1][0] : nullptr, log_blk.m_dual_plane ? &upsampled_weights[1][0] : nullptr); + + pUpsampled_weights = &upsampled_weights[0]; + } + } + + // Decode texels + const uint32_t ccs = log_blk.m_dual_plane ? log_blk.m_color_component_selector : UINT32_MAX; + + const uint8_t *pPart = &g_texel_partitions[log_blk.m_partition_id][0][0]; // [seed][y][x] + + const bool large_block = (num_blk_pixels >= 31); + uint32_t part_shift = (log_blk.m_num_partitions == 3) ? 2 : 0; + part_shift += large_block * 4; + + //uint32_t pixel_index = 0; + + if (log_blk.m_num_partitions == 1) + { + // alpha, 1 subset + int le0 = endpoints[0][0][0], he0 = endpoints[0][0][1]; + int le1 = endpoints[0][1][0], he1 = endpoints[0][1][1]; + int le2 = endpoints[0][2][0], he2 = endpoints[0][2][1]; + int le3 = endpoints[0][3][0], he3 = endpoints[0][3][1]; + + if (dec_mode == cDecodeModeSRGB8) + { + le0 = (le0 << 8) | 0x80; he0 = (he0 << 8) | 0x80; + le1 = (le1 << 8) | 0x80; he1 = (he1 << 8) | 0x80; + le2 = (le2 << 8) | 0x80; he2 = (he2 << 8) | 0x80; + le3 = (le3 << 8) | 0x80; he3 = (he3 << 8) | 0x80; + } + else + { + le0 = (le0 << 8) | le0; he0 = (he0 << 8) | he0; + le1 = (le1 << 8) | le1; he1 = (he1 << 8) | he1; + le2 = (le2 << 8) | le2; he2 = (he2 << 8) | he2; + le3 = (le3 << 8) | le3; he3 = (he3 << 8) | he3; + } + + // no subsets + if (!any_alpha) + { + if (!log_blk.m_dual_plane) + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + else + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + } + else // (!any_alpha) + { + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w3 = pUpsampled_weights[(3 == ccs) ? 1 : 0][pixel_index]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + const uint32_t k3 = weight_interpolate(le3, he3, w3); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = (uint8_t)(k3 >> 8); + + } // x + } // y + } + } + else + { + for (uint32_t subset = 0; subset < log_blk.m_num_partitions; subset++) + { + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + int le3 = endpoints[subset][3][0], he3 = endpoints[subset][3][1]; + + if (dec_mode == cDecodeModeSRGB8) + { + le0 = (le0 << 8) | 0x80; he0 = (he0 << 8) | 0x80; + le1 = (le1 << 8) | 0x80; he1 = (he1 << 8) | 0x80; + le2 = (le2 << 8) | 0x80; he2 = (he2 << 8) | 0x80; + le3 = (le3 << 8) | 0x80; he3 = (he3 << 8) | 0x80; + } + else + { + le0 = (le0 << 8) | le0; he0 = (he0 << 8) | he0; + le1 = (le1 << 8) | le1; he1 = (he1 << 8) | he1; + le2 = (le2 << 8) | le2; he2 = (he2 << 8) | he2; + le3 = (le3 << 8) | le3; he3 = (he3 << 8) | he3; + } + + endpoints[subset][0][0] = le0, endpoints[subset][0][1] = he0; + endpoints[subset][1][0] = le1, endpoints[subset][1][1] = he1; + endpoints[subset][2][0] = le2, endpoints[subset][2][1] = he2; + endpoints[subset][3][0] = le3, endpoints[subset][3][1] = he3; + } + + // subsets + if (!any_alpha) + { + // no alpha, sRGB + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t v = pPart[y * 12 + x]; + const uint32_t subset = (v >> part_shift) & 3; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = 255; + } // x + } // y + } + else + { + // alpha + for (uint32_t y = start_y; y < end_y; y++) + { + for (uint32_t x = start_x; x < end_x; x++) + { + const uint32_t pixel_index = x + y * blk_width; + + const uint32_t v = pPart[y * 12 + x]; + const uint32_t subset = (v >> part_shift) & 3; + + const uint32_t w0 = pUpsampled_weights[(0 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w1 = pUpsampled_weights[(1 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w2 = pUpsampled_weights[(2 == ccs) ? 1 : 0][pixel_index]; + const uint32_t w3 = pUpsampled_weights[(3 == ccs) ? 1 : 0][pixel_index]; + + int le0 = endpoints[subset][0][0], he0 = endpoints[subset][0][1]; + int le1 = endpoints[subset][1][0], he1 = endpoints[subset][1][1]; + int le2 = endpoints[subset][2][0], he2 = endpoints[subset][2][1]; + int le3 = endpoints[subset][3][0], he3 = endpoints[subset][3][1]; + + const uint32_t k0 = weight_interpolate(le0, he0, w0); + const uint32_t k1 = weight_interpolate(le1, he1, w1); + const uint32_t k2 = weight_interpolate(le2, he2, w2); + const uint32_t k3 = weight_interpolate(le3, he3, w3); + + ((uint8_t*)pPixels)[pixel_index * 4 + 0] = (uint8_t)(k0 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 1] = (uint8_t)(k1 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 2] = (uint8_t)(k2 >> 8); + ((uint8_t*)pPixels)[pixel_index * 4 + 3] = (uint8_t)(k3 >> 8); + + } // x + } // y + + } + + } // if (log_blk.m_num_partitions == 1) + + return true; + } + //------------------------------------------------ // Physical to logical block decoding @@ -3220,7 +4071,7 @@ namespace astc_helpers return *this; } }; - + static bool decode_void_extent(const uint128& bits, log_astc_block& log_blk) { if (bits.get_bits(10, 2) != 0b11) @@ -3232,9 +4083,9 @@ namespace astc_helpers const uint32_t min_t = bits.next_bits(bit_ofs, 13); const uint32_t max_t = bits.next_bits(bit_ofs, 13); assert(bit_ofs == 64); - + const bool all_extents_all_ones = (min_s == 0x1FFF) && (max_s == 0x1FFF) && (min_t == 0x1FFF) && (max_t == 0x1FFF); - + if (!all_extents_all_ones && ((min_s >= max_s) || (min_t >= max_t))) return false; @@ -3256,7 +4107,7 @@ namespace astc_helpers if (is_half_inf_or_nan(log_blk.m_solid_color[c])) return false; } - + return true; } @@ -3269,7 +4120,7 @@ namespace astc_helpers { // Dp_ofs, P_ofs, W_ofs, W_size, H_ofs, H_size, W_bias, H_bias, p0_ofs, p1_ofs, p2_ofs; { 10, 9, 7, 2, 5, 2, 4, 2, 4, 0, 1 }, // 4 2 - { 10, 9, 7, 2, 5, 2, 8, 2, 4, 0, 1 }, // 8 2 + { 10, 9, 7, 2, 5, 2, 8, 2, 4, 0, 1 }, // 8 2 { 10, 9, 5, 2, 7, 2, 2, 8, 4, 0, 1 }, // 2 8 { 10, 9, 5, 2, 7, 1, 2, 6, 4, 0, 1 }, // 2 6 @@ -3291,14 +4142,14 @@ namespace astc_helpers // Reserved if ((bits.get_bits(0, 2) == 0) && (bits.get_bits(6, 3) == 0b111)) { - if (bits.get_bits(2, 4) != 0b1111) + if (bits.get_bits(2, 4) != 0b1111) return false; } // Void extent if (bits.get_bits(0, 9) == 0b111111100) return decode_void_extent(bits, log_blk); - + // Check rows const uint32_t x0_2 = bits.get_bits(0, 2), x2_2 = bits.get_bits(2, 2); const uint32_t x5_4 = bits.get_bits(5, 4), x8_1 = bits.get_bits(8, 1); @@ -3344,7 +4195,7 @@ namespace astc_helpers if (r.Dp_ofs >= 0) Dp = bits.get_bits(r.Dp_ofs, 1) != 0; - + if (r.W_size) W += bits.get_bits(r.W_ofs, r.W_size); @@ -3353,7 +4204,7 @@ namespace astc_helpers assert((W >= MIN_GRID_DIM) && (W <= MAX_BLOCK_DIM)); assert((H >= MIN_GRID_DIM) && (H <= MAX_BLOCK_DIM)); - + int p0 = bits.get_bits(r.p0_ofs, 1); int p1 = bits.get_bits(r.p1_ofs, 1); int p2 = bits.get_bits(r.p2_ofs, 1); @@ -3361,10 +4212,10 @@ namespace astc_helpers uint32_t p = p0 | (p1 << 1) | (p2 << 2); if (p < 2) return false; - + log_blk.m_grid_width = (uint8_t)W; log_blk.m_grid_height = (uint8_t)H; - + log_blk.m_weight_ise_range = (uint8_t)((p - 2) + (P * BISE_10_LEVELS)); assert(log_blk.m_weight_ise_range <= LAST_VALID_WEIGHT_ISE_RANGE); @@ -3480,7 +4331,7 @@ namespace astc_helpers static void decode_bise(uint32_t ise_range, uint8_t* pVals, uint32_t num_vals, const uint128& bits, uint32_t bit_ofs) { assert(num_vals && (ise_range < TOTAL_ISE_RANGES)); - + const uint32_t bits_per_val = g_ise_range_table[ise_range][0]; if (g_ise_range_table[ise_range][1]) @@ -3521,24 +4372,24 @@ namespace astc_helpers return decode_bise(ise_range, pVals, num_vals, bits, bit_ofs); } - + // Decodes a physical ASTC block to a logical ASTC block. // blk_width/blk_height are only used to validate the weight grid's dimensions. bool unpack_block(const void* pASTC_block, log_astc_block& log_blk, uint32_t blk_width, uint32_t blk_height) { assert(is_valid_block_size(blk_width, blk_height)); - + const uint8_t* pS = (uint8_t*)pASTC_block; log_blk.clear(); log_blk.m_error_flag = true; - + const uint128 bits( (uint64_t)read_le_dword(pS) | (((uint64_t)read_le_dword(pS + sizeof(uint32_t))) << 32), (uint64_t)read_le_dword(pS + sizeof(uint32_t) * 2) | (((uint64_t)read_le_dword(pS + sizeof(uint32_t) * 3)) << 32)); - + const uint128 rev_bits(bits.get_reversed_bits()); - + if (!decode_config(bits, log_blk)) return false; @@ -3552,16 +4403,16 @@ namespace astc_helpers // Check grid dimensions if ((log_blk.m_grid_width > blk_width) || (log_blk.m_grid_height > blk_height)) return false; - + // Now we have the grid width/height, dual plane, weight ISE range - + const uint32_t total_grid_weights = (log_blk.m_dual_plane ? 2 : 1) * (log_blk.m_grid_width * log_blk.m_grid_height); const uint32_t total_weight_bits = get_ise_sequence_bits(total_grid_weights, log_blk.m_weight_ise_range); - + // 18.24 Illegal Encodings if ((!total_grid_weights) || (total_grid_weights > MAX_GRID_WEIGHTS) || (total_weight_bits < 24) || (total_weight_bits > 96)) return false; - + const uint32_t end_of_weight_bit_ofs = 128 - total_weight_bits; uint32_t total_extra_bits = 0; @@ -3598,9 +4449,9 @@ namespace astc_helpers return false; uint32_t cem_bit_pos = end_of_weight_bit_ofs - total_extra_bits; - + uint32_t c[4] = { 0 }, m[4] = { 0 }; - + cem_bits >>= 2; for (uint32_t i = 0; i < log_blk.m_num_partitions; i++, cem_bits >>= 1) c[i] = cem_bits & 1; @@ -3666,7 +4517,7 @@ namespace astc_helpers // config+num_parts+total_extra_bits (CEM extra+CCS) uint32_t total_config_bits = config_bit_pos + total_extra_bits; - + // Compute number of remaining bits in block const int num_remaining_bits = 128 - (int)total_config_bits - (int)total_weight_bits; if (num_remaining_bits < 0) @@ -3681,6 +4532,7 @@ namespace astc_helpers return false; // Infer endpoint ISE range based off the # of values we need to encode, and the # of remaining bits in the block + // TODO: Optimize int endpoint_ise_range = -1; for (int k = 20; k > 0; k--) { @@ -3709,6 +4561,289 @@ namespace astc_helpers return true; } + // Misc. helpers + + uint8_t get_weight(const log_astc_block& log_block, uint32_t plane_index, uint32_t i) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + assert(i < (uint32_t)(log_block.m_grid_width * log_block.m_grid_height)); + + const uint32_t idx = i * num_planes + plane_index; + assert(idx < MAX_GRID_WEIGHTS); + + return log_block.m_weights[idx]; + } + + uint8_t &get_weight(log_astc_block& log_block, uint32_t plane_index, uint32_t i) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + assert(i < (uint32_t)(log_block.m_grid_width * log_block.m_grid_height)); + + const uint32_t idx = i * num_planes + plane_index; + assert(idx < MAX_GRID_WEIGHTS); + + return log_block.m_weights[idx]; + } + + void extract_weights(const log_astc_block& log_block, uint8_t* pWeights, uint32_t plane_index) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + + const uint32_t num_weights = log_block.m_grid_width * log_block.m_grid_height; + for (uint32_t i = 0; i < num_weights; i++) + pWeights[i] = log_block.m_weights[i * num_planes + plane_index]; + } + + void set_weights(log_astc_block& log_block, const uint8_t* pWeights, uint32_t plane_index) + { + const uint32_t num_planes = log_block.m_dual_plane ? 2 : 1; + assert(plane_index < num_planes); + + const uint32_t num_weights = log_block.m_grid_width * log_block.m_grid_height; + for (uint32_t i = 0; i < num_weights; i++) + log_block.m_weights[i * num_planes + plane_index] = pWeights[i]; + } + + uint32_t get_total_weights(const log_astc_block& log_block) + { + return (log_block.m_dual_plane ? 2 : 1) * (log_block.m_grid_width * log_block.m_grid_height); + } + + // Returns a pointer to the beginning of a partition's/subset's endpoint values. + uint8_t *get_endpoints(log_astc_block& log_block, uint32_t partition_index) + { + assert(partition_index < log_block.m_num_partitions); + + uint32_t ofs = 0; + + for (uint32_t i = 0; i != partition_index; ++i) + ofs += get_num_cem_values(log_block.m_color_endpoint_modes[i]); + + assert(ofs < MAX_ENDPOINTS); + + return log_block.m_endpoints + ofs; + } + + const uint8_t* get_endpoints(const log_astc_block& log_block, uint32_t partition_index) + { + assert(partition_index < log_block.m_num_partitions); + + uint32_t ofs = 0; + + for (uint32_t i = 0; i != partition_index; ++i) + ofs += get_num_cem_values(log_block.m_color_endpoint_modes[i]); + + assert(ofs < MAX_ENDPOINTS); + + return log_block.m_endpoints + ofs; + } + + const char* get_cem_name(uint32_t cem_index) + { + static const char *s_cem_names[16] = + { + "CEM_LDR_LUM_DIRECT (0)", + "CEM_LDR_LUM_BASE_PLUS_OFS (1)", + "CEM_HDR_LUM_LARGE_RANGE (2)", + "CEM_HDR_LUM_SMALL_RANGE (3)", + "CEM_LDR_LUM_ALPHA_DIRECT (4)", + "CEM_LDR_LUM_ALPHA_BASE_PLUS_OFS (5)", + "CEM_LDR_RGB_BASE_SCALE (6)", + "CEM_HDR_RGB_BASE_SCALE (7)", + "CEM_LDR_RGB_DIRECT (8)", + "CEM_LDR_RGB_BASE_PLUS_OFFSET (9)", + "CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A (10)", + "CEM_HDR_RGB (11)", + "CEM_LDR_RGBA_DIRECT (12)", + "CEM_LDR_RGBA_BASE_PLUS_OFFSET (13)", + "CEM_HDR_RGB_LDR_ALPHA (14)", + "CEM_HDR_RGB_HDR_ALPHA (15)" + }; + + assert(cem_index < std::size(s_cem_names)); + const char *p = s_cem_names[cem_index]; + assert(p); + return p; + } + + bool cem_is_ldr_direct(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_DIRECT) || (cem_index == CEM_LDR_RGBA_DIRECT); + } + + bool cem_is_ldr_base_scale(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_BASE_SCALE) || (cem_index == CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A); + } + + bool cem_is_ldr_base_plus_ofs(uint32_t cem_index) + { + return (cem_index == CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == CEM_LDR_RGBA_BASE_PLUS_OFFSET); + } + + bool cem_supports_bc(uint32_t cem) + { + switch (cem) + { + case CEM_LDR_RGB_DIRECT: + case CEM_LDR_RGBA_DIRECT: + case CEM_LDR_RGB_BASE_PLUS_OFFSET: + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: + return true; + default: + break; + } + return false; + } + + // input: + // a=[0,255] + // b=[0,255] + // output: + // a=from, converted to -32 to 31 + // b=to, shifted right by 1 and 1 bit added to MSB, so [0,255] + void bit_transfer_signed_dec(int& a, int& b) + { + assert((a >= 0) && (a <= 255)); + assert((b >= 0) && (b <= 255)); + + b >>= 1; + b |= (a & 0x80); + + a >>= 1; + a &= 0x3F; + if ((a & 0x20) != 0) + a -= 0x40; + } + + // transfers a bit from b to a, prepares a for encoding + // input: + // a=[-32,31] (6-bits, 2's complement) + // b=[0,255] (8-bits) + // output: + // a=[0,255] (preserve top 2 bits) + // b=[0,255] + void bit_transfer_signed_enc(int& a, int& b) + { + assert((a >= -32) && (a <= 31)); + assert((b >= 0) && (b <= 255)); + + // extract MSB of b + bool bit_to_transfer = (b & 0x80) != 0; + b = (b << 1) & 0xFF; // 7 bits to 8 + + a &= 0x3F; // 6 bits + a <<= 1; // 6 to 7 bits + if (bit_to_transfer) + a |= 0x80; // set MSB + } + + // RGB or RGBA direct + bool cem8_or_12_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert((cem_index == CEM_LDR_RGB_DIRECT) || (cem_index == CEM_LDR_RGBA_DIRECT)); + (void)(cem_index); + + const auto& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[6]; + for (uint32_t i = 0; i < 6; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + uint32_t s0 = dequantized_endpoints[0] + dequantized_endpoints[2] + dequantized_endpoints[4]; + uint32_t s1 = dequantized_endpoints[1] + dequantized_endpoints[3] + dequantized_endpoints[5]; + + return s1 < s0; + } + + // RGB or RGBA base plus offset + bool cem9_or_13_used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert((cem_index == CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem_index == CEM_LDR_RGBA_BASE_PLUS_OFFSET)); + (void)(cem_index); + + const auto& endpoint_dequant_tab = g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + int dequantized_endpoints[6]; + for (uint32_t i = 0; i < 6; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + bit_transfer_signed_dec(dequantized_endpoints[1], dequantized_endpoints[0]); + bit_transfer_signed_dec(dequantized_endpoints[3], dequantized_endpoints[2]); + bit_transfer_signed_dec(dequantized_endpoints[5], dequantized_endpoints[4]); + + int s = dequantized_endpoints[1] + dequantized_endpoints[3] + dequantized_endpoints[5]; + + return s < 0; + } + + bool used_blue_contraction(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index) + { + assert(is_cem_ldr(cem_index)); + + bool used_blue_contraction_flag = false; + + if ((cem_index == 8) || (cem_index == 12)) + used_blue_contraction_flag = cem8_or_12_used_blue_contraction(cem_index, pEndpoint_vals, endpoint_ise_index); + else if ((cem_index == 9) || (cem_index == 13)) + used_blue_contraction_flag = cem9_or_13_used_blue_contraction(cem_index, pEndpoint_vals, endpoint_ise_index); + + return used_blue_contraction_flag; + } + + uint32_t get_base_cem_without_alpha(uint32_t cem) + { + assert(is_cem_ldr(cem)); + + switch (cem) + { + case CEM_LDR_LUM_ALPHA_DIRECT: return CEM_LDR_LUM_DIRECT; + case CEM_LDR_RGBA_DIRECT: return CEM_LDR_RGB_DIRECT; + case CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: return CEM_LDR_RGB_BASE_SCALE; + case CEM_LDR_RGBA_BASE_PLUS_OFFSET: return CEM_LDR_RGB_BASE_PLUS_OFFSET; + default: + break; + } + + return cem; + } + + int apply_delta_to_bise_endpoint_val(uint32_t endpoint_ise_range, int ise_val, int delta) + { + if (delta == 0) + return ise_val; + + uint32_t num_ise_levels = astc_helpers::get_ise_levels(endpoint_ise_range); + + const auto& ISE_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_ISE_to_rank; + const auto& rank_to_ISE = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_range).m_rank_to_ISE; + + int cur_rank = ISE_to_rank[ise_val]; + int new_rank = basisu::clamp(cur_rank + delta, 0, (int)num_ise_levels - 1); + + return rank_to_ISE[new_rank]; + } + + void get_astc_block_size_by_index(uint32_t index, uint32_t& width, uint32_t& height) + { + assert(index < NUM_ASTC_BLOCK_SIZES); + + width = g_astc_block_sizes[index][0]; + height = g_astc_block_sizes[index][1]; + } + + int find_astc_block_size_index(uint32_t width, uint32_t height) + { + for (uint32_t i = 0; i < NUM_ASTC_BLOCK_SIZES; i++) + if ((width == g_astc_block_sizes[i][0]) && (height == g_astc_block_sizes[i][1])) + return i; + + return -1; + } + } // namespace astc_helpers #endif //BASISU_ASTC_HELPERS_IMPLEMENTATION diff --git a/transcoder/basisu_containers.h b/transcoder/basisu_containers.h index 88026c7..dc816f3 100644 --- a/transcoder/basisu_containers.h +++ b/transcoder/basisu_containers.h @@ -159,8 +159,8 @@ namespace basisu static inline void construct(T** p) { memset(p, 0, sizeof(T*)); } static inline void construct(T** p, T* init) { *p = init; } static inline void construct_array(T** p, size_t n) { memset(p, 0, sizeof(T*) * n); } - static inline void destruct(T** p) { p; } - static inline void destruct_array(T** p, size_t n) { p, n; } + static inline void destruct(T** p) { (void)p; } + static inline void destruct_array(T** p, size_t n) { (void)p, (void)n; } }; #define BASISU_DEFINE_BUILT_IN_TYPE(X) \ @@ -169,8 +169,8 @@ namespace basisu static inline void construct(X* p) { memset(p, 0, sizeof(X)); } \ static inline void construct(X* p, const X& init) { memcpy(p, &init, sizeof(X)); } \ static inline void construct_array(X* p, size_t n) { memset(p, 0, sizeof(X) * n); } \ - static inline void destruct(X* p) { p; } \ - static inline void destruct_array(X* p, size_t n) { p, n; } }; + static inline void destruct(X* p) { (void)p; } \ + static inline void destruct_array(X* p, size_t n) { (void)p, (void)n; } }; BASISU_DEFINE_BUILT_IN_TYPE(bool) BASISU_DEFINE_BUILT_IN_TYPE(char) @@ -272,7 +272,7 @@ namespace basisu size_t c = a + b; return c < a; } - + // Returns false on overflow, true if OK. template inline bool can_fit_into_size_t(T val) @@ -294,7 +294,7 @@ namespace basisu template class writable_span; - + template class readable_span { @@ -304,7 +304,7 @@ namespace basisu using const_pointer = const T*; using const_reference = const T&; using const_iterator = const T*; - + inline readable_span() : m_p(nullptr), m_size(0) @@ -941,7 +941,7 @@ namespace basisu inline iterator begin() const { return m_p; } inline iterator end() const { assert(m_p || !m_size); return m_p + m_size; } - + inline const_iterator cbegin() const { return m_p; } inline const_iterator cend() const { assert(m_p || !m_size); return m_p + m_size; } @@ -1506,7 +1506,7 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif if ((m_p) && (other.m_p)) { @@ -1563,6 +1563,52 @@ namespace basisu set(ws); } + // mostly to ease porting from std::vector, not particularly optimized + inline void assign(size_t new_size, const T& init) + { + assert(!m_p || (&init < m_p) || (&init >= (m_p + m_size))); + + // Blow away existing contents + resize(0); + + if (new_size) + { + resize(new_size); + + for (size_t i = 0; i < new_size; ++i) + m_p[i] = init; + } + } + + // mostly to ease porting from std::vector, not particularly optimized + template + inline void assign(const R* pBegin, const R* pEnd) + { + assert(!m_p || + (reinterpret_cast(pEnd) <= reinterpret_cast(m_p)) || + (reinterpret_cast(pBegin) >= reinterpret_cast(m_p + m_size)) + ); + + // Blow away existing contents + resize(0); + + if ((!pBegin) || (!pEnd) || (pEnd <= pBegin)) + { + assert(0); + return; + } + + const size_t new_size = static_cast(static_cast(pEnd - pBegin)); + + if (new_size) + { + resize(new_size); + + for (size_t i = 0; i < new_size; ++i) + m_p[i] = static_cast(*pBegin++); + } + } + // Set contents of vector to contents of the readable span bool set(const readable_span& rs) { @@ -1647,7 +1693,7 @@ namespace basisu { #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif if ((m_p) && (other.m_p)) memcpy((void *)m_p, other.m_p, other.m_size * sizeof(T)); @@ -2147,7 +2193,7 @@ namespace basisu if (!try_insert(p, obj)) container_abort("vector::insert() failed!\n"); } - + // push_front() isn't going to be very fast - it's only here for usability. inline void push_front(const T& obj) { @@ -2228,7 +2274,7 @@ namespace basisu #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif memmove((void *)pDst, pSrc, num_to_move * sizeof(T)); @@ -2239,7 +2285,7 @@ namespace basisu } else { - // Type is not bitwise copyable or movable. + // Type is not bitwise copyable or movable. // Move them down one at a time by using the equals operator, and destroying anything that's left over at the end. T* pDst_end = pDst + num_to_move; @@ -2482,7 +2528,7 @@ namespace basisu { #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wclass-memaccess" #endif memset(m_p, *reinterpret_cast(&o), m_size); @@ -2770,6 +2816,7 @@ namespace basisu m_grow_threshold = 0; } + // Destroys elements/empties container but doesn't free memory. inline void reset() { if (!m_num_valid) @@ -2798,7 +2845,7 @@ namespace basisu } else if (sizeof(node) <= 16) { - memset(&m_values[0], 0, m_values.size_in_bytes()); + memset((void *)&m_values[0], 0, m_values.size_in_bytes()); } else { @@ -2829,6 +2876,11 @@ namespace basisu return m_num_valid; } + inline uint32_t size_u32() + { + return static_cast(m_num_valid); + } + inline size_t get_table_size() { return m_values.size(); @@ -3102,7 +3154,7 @@ namespace basisu { return try_insert(result, std::move(v.first), std::move(v.second)); } - + inline const_iterator find(const Key& k) const { return const_iterator(*this, find_index(k)); @@ -3183,12 +3235,12 @@ namespace basisu static inline void construct_value_type(value_type* pDst, const Key& k, const Value& v) { if (BASISU_IS_BITWISE_COPYABLE(Key)) - memcpy(&pDst->first, &k, sizeof(Key)); + memcpy((void *)&pDst->first, &k, sizeof(Key)); else scalar_type::construct(&pDst->first, k); if (BASISU_IS_BITWISE_COPYABLE(Value)) - memcpy(&pDst->second, &v, sizeof(Value)); + memcpy((void *)&pDst->second, &v, sizeof(Value)); else scalar_type::construct(&pDst->second, v); } @@ -3197,17 +3249,17 @@ namespace basisu { if ((BASISU_IS_BITWISE_COPYABLE(Key)) && (BASISU_IS_BITWISE_COPYABLE(Value))) { - memcpy(pDst, pSrc, sizeof(value_type)); + memcpy((void *)pDst, pSrc, sizeof(value_type)); } else { if (BASISU_IS_BITWISE_COPYABLE(Key)) - memcpy(&pDst->first, &pSrc->first, sizeof(Key)); + memcpy((void *)&pDst->first, &pSrc->first, sizeof(Key)); else scalar_type::construct(&pDst->first, pSrc->first); if (BASISU_IS_BITWISE_COPYABLE(Value)) - memcpy(&pDst->second, &pSrc->second, sizeof(Value)); + memcpy((void *)&pDst->second, &pSrc->second, sizeof(Value)); else scalar_type::construct(&pDst->second, pSrc->second); } @@ -3227,14 +3279,14 @@ namespace basisu if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Key) && BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Value)) { - memcpy(pDst, pSrc, sizeof(node)); + memcpy((void *)pDst, pSrc, sizeof(node)); assert(pDst->state == cStateValid); } else { if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Key)) - memcpy(&pDst->first, &pSrc->first, sizeof(Key)); + memcpy((void*)&pDst->first, &pSrc->first, sizeof(Key)); else { new ((void*)&pDst->first) Key(std::move(pSrc->first)); @@ -3242,7 +3294,7 @@ namespace basisu } if (BASISU_IS_BITWISE_COPYABLE_OR_MOVABLE(Value)) - memcpy(&pDst->second, &pSrc->second, sizeof(Value)); + memcpy((void*)&pDst->second, &pSrc->second, sizeof(Value)); else { new ((void*)&pDst->second) Value(std::move(pSrc->second)); @@ -3583,7 +3635,7 @@ namespace basisu // Not checking for is MOVABLE because the caller could later destruct k and/or v (what state do we set them to?) if (BASISU_IS_BITWISE_COPYABLE(Key)) { - memcpy(&pDst->first, &k, sizeof(Key)); + memcpy((void *)&pDst->first, (const void *)&k, sizeof(Key)); } else { @@ -3593,7 +3645,7 @@ namespace basisu if (BASISU_IS_BITWISE_COPYABLE(Value)) { - memcpy(&pDst->second, &v, sizeof(Value)); + memcpy((void *)&pDst->second, (const void*)&v, sizeof(Value)); } else { @@ -3721,11 +3773,11 @@ namespace basisu va_list args; va_start(args, pFmt); -#ifdef _WIN32 +#ifdef _WIN32 vsprintf_s(buf, sizeof(buf), pFmt, args); #else vsnprintf(buf, sizeof(buf), pFmt, args); -#endif +#endif va_end(args); return std::string(buf); @@ -3893,7 +3945,7 @@ namespace basisu std::size_t copy_size = std::min(list.size(), N); std::copy_n(list.begin(), copy_size, m_data); // Copy up to min(list.size(), N) - if (list.size() < N) + if (list.size() < N) { // Initialize the rest of the array std::fill(m_data + copy_size, m_data + N, T{}); @@ -3907,7 +3959,7 @@ namespace basisu return m_data[index]; } - BASISU_FORCE_INLINE const T& operator[](std::size_t index) const + BASISU_FORCE_INLINE const T& operator[](std::size_t index) const { if (index >= N) container_abort("fixed_array: Index out of bounds."); @@ -3950,26 +4002,26 @@ namespace basisu { return writable_span(m_data, N); } - + private: BASISU_FORCE_INLINE void initialize_array() { - if constexpr (std::is_integral::value || std::is_floating_point::value) + if constexpr (std::is_integral::value || std::is_floating_point::value) memset(m_data, 0, sizeof(m_data)); - else + else std::fill(m_data, m_data + N, T{}); } BASISU_FORCE_INLINE T& access_element(std::size_t index) { - if (index >= N) + if (index >= N) container_abort("fixed_array: Index out of bounds."); return m_data[index]; } BASISU_FORCE_INLINE const T& access_element(std::size_t index) const { - if (index >= N) + if (index >= N) container_abort("fixed_array: Index out of bounds."); return m_data[index]; } @@ -4046,6 +4098,9 @@ namespace basisu inline uint32_t get_width() const { return m_width; } inline uint32_t get_height() const { return m_height; } + inline uint32_t get_cols() const { return m_width; } + inline uint32_t get_rows() const { return m_height; } + inline const T& operator() (uint32_t x, uint32_t y) const { assert(x < m_width && y < m_height); return m_values[x + y * m_width]; } inline T& operator() (uint32_t x, uint32_t y) { assert(x < m_width && y < m_height); return m_values[x + y * m_width]; } @@ -4054,9 +4109,23 @@ namespace basisu inline const T& operator[] (uint32_t i) const { return m_values[i]; } inline T& operator[] (uint32_t i) { return m_values[i]; } + inline const T& at(int x, int y) const { return (*this)((uint32_t)x, (uint32_t)y); } + inline T& at(int x, int y) { return (*this)((uint32_t)x, (uint32_t)y); } + inline const T& at_clamped(int x, int y) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } inline T& at_clamped(int x, int y) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline const T& at_row_col(int y, int x) const { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + inline T& at_row_col(int y, int x) { return (*this)(clamp(x, 0, m_width - 1), clamp(y, 0, m_height - 1)); } + + inline void set_clipped(int x, int y, const T& val) + { + if ( ((uint32_t)x >= m_width) || ((uint32_t)y >= m_height) ) + return; + + m_values[x + y * m_width] = val; + } + void clear() { m_width = 0; @@ -4141,9 +4210,18 @@ namespace basisu return true; } + vector2D& resize_rows_cols(uint32_t rows, uint32_t cols) + { + return resize(cols, rows); + } + + bool try_resize_rows_cols(uint32_t rows, uint32_t cols) + { + return try_resize(cols, rows); + } + const vector2D& extract_block_clamped(T* pDst, uint32_t src_x, uint32_t src_y, uint32_t w, uint32_t h) const { - // HACK HACK if (((src_x + w) > m_width) || ((src_y + h) > m_height)) { // Slower clamping case @@ -4165,8 +4243,87 @@ namespace basisu return *this; } + + const vector2D& extract_block_clamped(T* pDst, uint32_t src_x, uint32_t src_y, uint32_t w, uint32_t h, uint32_t override_height) const + { + assert(override_height && (override_height <= m_height)); + + if (((src_x + w) > m_width) || ((src_y + h) > minimum(m_height, override_height))) + { + // Slower clamping case + for (uint32_t y = 0; y < h; y++) + for (uint32_t x = 0; x < w; x++) + *pDst++ = at_clamped(src_x + x, minimum(src_y + y, override_height - 1)); + } + else + { + const T* pSrc = &m_values[src_x + src_y * m_width]; + + for (uint32_t y = 0; y < h; y++) + { + memcpy(pDst, pSrc, w * sizeof(T)); + pSrc += m_width; + pDst += w; + } + } + + return *this; + } }; + // Explictly primitive container intended for POD's, simple usage. + // push_back() and resize() will refuse to push anymore and just return when full. + template + class static_vector + { + T m_data[N]; + uint32_t m_size; + + public: + static_vector() : m_size(0) { } + + inline void reserve(size_t reserve_size) + { + (void)(reserve_size); + + assert(reserve_size <= N); + } + + inline void push_back(const T& value) + { + // Should never happen. + if (m_size >= N) + { + assert(0); + fprintf(stderr, "basisu::static_vector overflow!\n"); + return; + } + + m_data[m_size++] = value; + } + + inline std::size_t size() const { return m_size; } + inline uint32_t size_u32() const { return m_size; } + inline constexpr std::size_t capacity() const { return N; } + + inline bool empty() const { return !m_size; } + + inline T& operator[](std::size_t i) { return m_data[i]; } + inline const T& operator[](std::size_t i) const { return m_data[i]; } + + inline void resize(size_t new_size) + { + if (new_size > N) + { + assert(0); + fprintf(stderr, "basisu::static_vector overflow!\n"); + return; + } + + m_size = (uint32_t)new_size; + } + }; + } // namespace basisu namespace std diff --git a/transcoder/basisu_containers_impl.h b/transcoder/basisu_containers_impl.h index 4c85ed3..2ac1302 100644 --- a/transcoder/basisu_containers_impl.h +++ b/transcoder/basisu_containers_impl.h @@ -14,7 +14,7 @@ namespace basisu #ifdef _MSC_VER __declspec(noreturn) #else - [[noreturn]] + [[noreturn]] #endif void container_abort(const char* pMsg, ...) { @@ -42,12 +42,12 @@ namespace basisu assert(m_size <= m_capacity); assert(min_new_capacity >= m_size); assert(element_size); - + // Basic sanity check min_new_capacity if (!can_fit_into_size_t((uint64_t)min_new_capacity * element_size)) { assert(0); - + if (nofail_flag) return false; @@ -100,7 +100,7 @@ namespace basisu } const size_t desired_size = static_cast(desired_size_u64); - + size_t actual_size = 0; BASISU_NOTE_UNUSED(actual_size); @@ -109,6 +109,7 @@ namespace basisu void* new_p = realloc(m_p, desired_size); if (!new_p) { + fprintf(stderr, "elemental_vector::increase_capacity: Allocation failed!\n"); assert(0); if (nofail_flag) @@ -133,7 +134,9 @@ namespace basisu void* new_p = malloc(desired_size); if (!new_p) { + fprintf(stderr, "elemental_vector::increase_capacity: Allocation failed!\n"); assert(0); + if (nofail_flag) return false; @@ -269,7 +272,7 @@ namespace basisu s.insert(i); k.push_back(i); } - + for (uint32_t i = 0; i < k.size(); i++) { uint32_t r = rand() ^ (rand() << 15); @@ -315,7 +318,7 @@ namespace basisu { typedef basisu::hash_map< uint32_t, basisu::vector > hm; hm q; - + basisu::vector a, b; a.push_back(1); b.push_back(2); diff --git a/transcoder/basisu_file_headers.h b/transcoder/basisu_file_headers.h index ddd117a..1046277 100644 --- a/transcoder/basisu_file_headers.h +++ b/transcoder/basisu_file_headers.h @@ -1,5 +1,5 @@ // basis_file_headers.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ namespace basist basisu::packed_uint<2> m_orig_height; // The original image height (may not be a multiple of 4 pixels) basisu::packed_uint<2> m_num_blocks_x; // The slice's block X dimensions. Each block is 4x4 or 6x6 pixels. The slice's pixel resolution may or may not be a power of 2. - basisu::packed_uint<2> m_num_blocks_y; // The slice's block Y dimensions. + basisu::packed_uint<2> m_num_blocks_y; // The slice's block Y dimensions. basisu::packed_uint<4> m_file_ofs; // Offset from the start of the file to the start of the slice's data basisu::packed_uint<4> m_file_size; // The size of the compressed slice data in bytes @@ -59,16 +59,16 @@ namespace basist // Set if any slices contain alpha (for ETC1S, if the odd slices contain alpha data) cBASISHeaderFlagHasAlphaSlices = 4, - // For ETC1S files, this will be true if the file utilizes a codebook from another .basis file. + // For ETC1S files, this will be true if the file utilizes a codebook from another .basis file. cBASISHeaderFlagUsesGlobalCodebook = 8, - // Set if the texture data is sRGB, otherwise it's linear. + // Set if the texture data is sRGB, otherwise it's linear. // In reality, we have no idea if the texture data is actually linear or sRGB. This is the m_perceptual parameter passed to the compressor. cBASISHeaderFlagSRGB = 16, }; // The image type field attempts to describe how to interpret the image data in a Basis file. - // The encoder library doesn't really do anything special or different with these texture types, this is mostly here for the benefit of the user. + // The encoder library doesn't really do anything special or different with these texture types, this is mostly here for the benefit of the user. // We do make sure the various constraints are followed (2DArray/cubemap/videoframes/volume implies that each image has the same resolution and # of mipmap levels, etc., cubemap implies that the # of image slices is a multiple of 6) enum basis_texture_type { @@ -88,14 +88,113 @@ namespace basist enum class basis_tex_format { + // Original LDR formats cETC1S = 0, - cUASTC4x4 = 1, + cUASTC_LDR_4x4 = 1, + + // HDR formats cUASTC_HDR_4x4 = 2, cASTC_HDR_6x6 = 3, - cASTC_HDR_6x6_INTERMEDIATE = 4, + cUASTC_HDR_6x6_INTERMEDIATE = 4, // TODO: rename to UASTC_HDR_6x6 + + // XUASTC (supercompressed) LDR variants (the standard ASTC block sizes) + cXUASTC_LDR_4x4 = 5, + cXUASTC_LDR_5x4 = 6, + cXUASTC_LDR_5x5 = 7, + cXUASTC_LDR_6x5 = 8, + + cXUASTC_LDR_6x6 = 9, + cXUASTC_LDR_8x5 = 10, + cXUASTC_LDR_8x6 = 11, + cXUASTC_LDR_10x5 = 12, + + cXUASTC_LDR_10x6 = 13, + cXUASTC_LDR_8x8 = 14, + cXUASTC_LDR_10x8 = 15, + cXUASTC_LDR_10x10 = 16, + + cXUASTC_LDR_12x10 = 17, + cXUASTC_LDR_12x12 = 18, + + // Standard (non-supercompressed) ASTC LDR variants (the standard ASTC block sizes) + cASTC_LDR_4x4 = 19, + cASTC_LDR_5x4 = 20, + cASTC_LDR_5x5 = 21, + cASTC_LDR_6x5 = 22, + + cASTC_LDR_6x6 = 23, + cASTC_LDR_8x5 = 24, + cASTC_LDR_8x6 = 25, + cASTC_LDR_10x5 = 26, + + cASTC_LDR_10x6 = 27, + cASTC_LDR_8x8 = 28, + cASTC_LDR_10x8 = 29, + cASTC_LDR_10x10 = 30, + + cASTC_LDR_12x10 = 31, + cASTC_LDR_12x12 = 32, + cTotalFormats }; + // True if the basis_tex_format is XUASTC LDR 4x4-12x12. + inline bool basis_tex_format_is_xuastc_ldr(basis_tex_format tex_fmt) + { + return ((uint32_t)tex_fmt >= (uint32_t)basis_tex_format::cXUASTC_LDR_4x4) && ((uint32_t)tex_fmt <= (uint32_t)basis_tex_format::cXUASTC_LDR_12x12); + } + + // True if the basis_tex_format is ASTC LDR 4x4-12x12. + inline bool basis_tex_format_is_astc_ldr(basis_tex_format tex_fmt) + { + return ((uint32_t)tex_fmt >= (uint32_t)basis_tex_format::cASTC_LDR_4x4) && ((uint32_t)tex_fmt <= (uint32_t)basis_tex_format::cASTC_LDR_12x12); + } + + inline void get_basis_tex_format_block_size(basis_tex_format tex_fmt, uint32_t &width, uint32_t &height) + { + switch (tex_fmt) + { + case basis_tex_format::cETC1S: width = 4; height = 4; break; + case basis_tex_format::cUASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cUASTC_HDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cASTC_HDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: width = 6; height = 6; break; + case basis_tex_format::cXUASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cXUASTC_LDR_5x4: width = 5; height = 4; break; + case basis_tex_format::cXUASTC_LDR_5x5: width = 5; height = 5; break; + case basis_tex_format::cXUASTC_LDR_6x5: width = 6; height = 5; break; + case basis_tex_format::cXUASTC_LDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cXUASTC_LDR_8x5: width = 8; height = 5; break; + case basis_tex_format::cXUASTC_LDR_8x6: width = 8; height = 6; break; + case basis_tex_format::cXUASTC_LDR_10x5: width = 10; height = 5; break; + case basis_tex_format::cXUASTC_LDR_10x6: width = 10; height = 6; break; + case basis_tex_format::cXUASTC_LDR_8x8: width = 8; height = 8; break; + case basis_tex_format::cXUASTC_LDR_10x8: width = 10; height = 8; break; + case basis_tex_format::cXUASTC_LDR_10x10: width = 10; height = 10; break; + case basis_tex_format::cXUASTC_LDR_12x10: width = 12; height = 10; break; + case basis_tex_format::cXUASTC_LDR_12x12: width = 12; height = 12; break; + case basis_tex_format::cASTC_LDR_4x4: width = 4; height = 4; break; + case basis_tex_format::cASTC_LDR_5x4: width = 5; height = 4; break; + case basis_tex_format::cASTC_LDR_5x5: width = 5; height = 5; break; + case basis_tex_format::cASTC_LDR_6x5: width = 6; height = 5; break; + case basis_tex_format::cASTC_LDR_6x6: width = 6; height = 6; break; + case basis_tex_format::cASTC_LDR_8x5: width = 8; height = 5; break; + case basis_tex_format::cASTC_LDR_8x6: width = 8; height = 6; break; + case basis_tex_format::cASTC_LDR_10x5: width = 10; height = 5; break; + case basis_tex_format::cASTC_LDR_10x6: width = 10; height = 6; break; + case basis_tex_format::cASTC_LDR_8x8: width = 8; height = 8; break; + case basis_tex_format::cASTC_LDR_10x8: width = 10; height = 8; break; + case basis_tex_format::cASTC_LDR_10x10: width = 10; height = 10; break; + case basis_tex_format::cASTC_LDR_12x10: width = 12; height = 10; break; + case basis_tex_format::cASTC_LDR_12x12: width = 12; height = 12; break; + default: + assert(0); + width = 0; + height = 0; + break; + } + } + struct basis_file_header { enum @@ -115,7 +214,7 @@ namespace basist basisu::packed_uint<3> m_total_slices; // The total # of compressed slices (1 slice per image, or 2 for alpha .basis files) basisu::packed_uint<3> m_total_images; // The total # of images - + basisu::packed_uint<1> m_tex_format; // enum basis_tex_format basisu::packed_uint<2> m_flags; // enum basist::header_flags basisu::packed_uint<1> m_tex_type; // enum basist::basis_texture_type @@ -125,11 +224,11 @@ namespace basist basisu::packed_uint<4> m_userdata0; // For client use basisu::packed_uint<4> m_userdata1; // For client use - basisu::packed_uint<2> m_total_endpoints; // The number of endpoints in the endpoint codebook + basisu::packed_uint<2> m_total_endpoints; // The number of endpoints in the endpoint codebook basisu::packed_uint<4> m_endpoint_cb_file_ofs; // The compressed endpoint codebook's file offset relative to the start of the file basisu::packed_uint<3> m_endpoint_cb_file_size; // The compressed endpoint codebook's size in bytes - basisu::packed_uint<2> m_total_selectors; // The number of selectors in the endpoint codebook + basisu::packed_uint<2> m_total_selectors; // The number of selectors in the endpoint codebook basisu::packed_uint<4> m_selector_cb_file_ofs; // The compressed selectors codebook's file offset relative to the start of the file basisu::packed_uint<3> m_selector_cb_file_size; // The compressed selector codebook's size in bytes @@ -137,7 +236,7 @@ namespace basist basisu::packed_uint<4> m_tables_file_size; // The file size in bytes of the compressed huffman codelength tables basisu::packed_uint<4> m_slice_desc_file_ofs; // The file offset to the slice description array, usually follows the header - + basisu::packed_uint<4> m_extended_file_ofs; // The file offset of the "extended" header and compressed data, for future use basisu::packed_uint<4> m_extended_file_size; // The file size in bytes of the "extended" header and compressed data, for future use }; diff --git a/transcoder/basisu_transcoder.cpp b/transcoder/basisu_transcoder.cpp index f55d34e..347a024 100644 --- a/transcoder/basisu_transcoder.cpp +++ b/transcoder/basisu_transcoder.cpp @@ -1,5 +1,5 @@ -// basisu_transcoder.cpp -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// basisu_transcoder.cpp +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -107,6 +107,10 @@ #define BASISD_SUPPORT_ASTC 1 #endif +#ifndef BASISD_SUPPORT_XUASTC +#define BASISD_SUPPORT_XUASTC 1 +#endif + // Note that if BASISD_SUPPORT_ATC is enabled, BASISD_SUPPORT_DXT5A should also be enabled for alpha support. #ifndef BASISD_SUPPORT_ATC #define BASISD_SUPPORT_ATC 1 @@ -161,6 +165,7 @@ #define BASISD_WRITE_NEW_ETC2_EAC_R11_TABLES 0 #ifndef BASISD_ENABLE_DEBUG_FLAGS + // DO NOT CHECK IN #define BASISD_ENABLE_DEBUG_FLAGS 0 #endif @@ -177,6 +182,14 @@ using namespace basist::astc_6x6_hdr; #endif +#if BASISD_IS_BIG_ENDIAN +const uint32_t BASISD_COLOR_RGBA_A_MASK = 0x000000FF; +const uint32_t BASISD_COLOR_RGBA_RGB_MASK = ~BASISD_COLOR_RGBA_A_MASK; +#else +const uint32_t BASISD_COLOR_RGBA_A_MASK = 0xFF000000; +const uint32_t BASISD_COLOR_RGBA_RGB_MASK = ~BASISD_COLOR_RGBA_A_MASK; +#endif + namespace basisu { bool g_debug_printf; @@ -188,7 +201,7 @@ namespace basisu void debug_printf(const char* pFmt, ...) { -#if BASISU_FORCE_DEVEL_MESSAGES +#if BASISU_FORCE_DEVEL_MESSAGES g_debug_printf = true; #endif if (g_debug_printf) @@ -202,7 +215,7 @@ namespace basisu void debug_puts(const char* p) { -#if BASISU_FORCE_DEVEL_MESSAGES +#if BASISU_FORCE_DEVEL_MESSAGES g_debug_printf = true; #endif if (g_debug_printf) @@ -235,6 +248,14 @@ namespace basist g_debug_flags = f; #endif } + + // Used by arith encoder/decoder + namespace arith_fastbits_f32 + { + bool g_initialized; + float g_lut_edge[TABLE_SIZE + 1]; // samples at m = 1 + i/TABLE_SIZE (for linear) + + } // namespace arith_fastbits_f32 inline uint16_t byteswap_uint16(uint16_t v) { @@ -312,6 +333,8 @@ namespace basist res[i] = basisu::lerp(a[i], b[i], s); return res; } + + inline float norm() const { return dot(*this); } }; uint16_t crc16(const void* r, size_t size, uint16_t crc) @@ -329,6 +352,62 @@ namespace basist return static_cast(~crc); } + uint32_t hash_hsieh(const uint8_t* pBuf, size_t len) + { + if (!pBuf || !len) + return 0; + + uint32_t h = static_cast(len); + + const uint32_t bytes_left = len & 3; + len >>= 2; + + while (len--) + { + const uint16_t* pWords = reinterpret_cast(pBuf); + + h += pWords[0]; + + const uint32_t t = (pWords[1] << 11) ^ h; + h = (h << 16) ^ t; + + pBuf += sizeof(uint32_t); + + h += h >> 11; + } + + switch (bytes_left) + { + case 1: + h += *reinterpret_cast(pBuf); + h ^= h << 10; + h += h >> 1; + break; + case 2: + h += *reinterpret_cast(pBuf); + h ^= h << 11; + h += h >> 17; + break; + case 3: + h += *reinterpret_cast(pBuf); + h ^= h << 16; + h ^= (static_cast(pBuf[sizeof(uint16_t)])) << 18; + h += h >> 11; + break; + default: + break; + } + + h ^= h << 3; + h += h >> 5; + h ^= h << 4; + h += h >> 17; + h ^= h << 25; + h += h >> 6; + + return h; + } + struct vec4F { float c[4]; @@ -338,7 +417,7 @@ namespace basist float operator[] (uint32_t index) const { assert(index < 4); return c[index]; } float& operator[] (uint32_t index) { assert(index < 4); return c[index]; } }; - + enum etc_constants { cETC1BytesPerBlock = 8U, @@ -411,14 +490,14 @@ namespace basist //const uint8_t g_etc1_to_selector_index[cETC1SelectorValues] = { 2, 3, 1, 0 }; const uint8_t g_selector_index_to_etc1[cETC1SelectorValues] = { 3, 2, 0, 1 }; - + static const uint8_t g_etc_5_to_8[32] = { 0, 8, 16, 24, 33, 41, 49, 57, 66, 74, 82, 90, 99, 107, 115, 123, 132, 140, 148, 156, 165, 173, 181, 189, 198, 206, 214, 222, 231, 239, 247, 255 }; struct decoder_etc_block { // big endian uint64: // bit ofs: 56 48 40 32 24 16 8 0 - // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7 + // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7 union { uint64_t m_uint64; @@ -686,7 +765,7 @@ namespace basist { return (m_bytes[3] & 2) != 0; } - + inline uint32_t get_inten_table(uint32_t subblock_id) const { assert(subblock_id < 2); @@ -701,7 +780,7 @@ namespace basist const uint32_t b = get_byte_bits(cETC1DeltaColor3BBitOffset, 3); return static_cast(b | (g << 3U) | (r << 6U)); } - + void get_block_colors(color32* pBlock_colors, uint32_t subblock_index) const { color32 b; @@ -819,7 +898,7 @@ namespace basist g = c.g; b = c.b; } - + static void unpack_color5(color32& result, uint16_t packed_color5, bool scaled) { result = unpack_color5(packed_color5, scaled, 255); @@ -948,7 +1027,7 @@ namespace basist static void get_block_color5_r(const color32& base_color5, uint32_t inten_table, uint32_t index, uint32_t &r) { assert(index < 4); - + uint32_t br = (base_color5.r << 3) | (base_color5.r >> 2); const int* pInten_table = g_etc1_inten_tables[inten_table]; @@ -1124,7 +1203,7 @@ namespace basist { 1, 2, 2, 2 }, { 1, 2, 3, 3 }, }; - + static uint8_t g_etc1_to_dxt1_selector_mappings_raw_dxt1_256[NUM_ETC1_TO_DXT1_SELECTOR_MAPPINGS][256]; static uint8_t g_etc1_to_dxt1_selector_mappings_raw_dxt1_inv_256[NUM_ETC1_TO_DXT1_SELECTOR_MAPPINGS][256]; @@ -1515,9 +1594,9 @@ namespace basist return best_err; } #endif // BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES - + static -#if !BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES +#if !BASISD_WRITE_NEW_ETC2_EAC_A8_TABLES const #endif etc1_g_to_eac_conversion s_etc1_g_to_etc2_a8[32 * 8][NUM_ETC2_EAC_SELECTOR_RANGES] = @@ -2006,7 +2085,7 @@ namespace basist void uastc_init(); #endif -#if BASISD_SUPPORT_UASTC_HDR +#if BASISD_SUPPORT_UASTC_HDR namespace astc_6x6_hdr { static void init_quantize_tables(); @@ -2021,19 +2100,24 @@ namespace basist } #endif - static bool g_transcoder_initialized; + namespace astc_ldr_t + { + void init_transcoding_tables(); + } + static bool g_transcoder_initialized; + // Library global initialization. Requires ~9 milliseconds when compiled and executed natively on a Core i7 2.2 GHz. // If this is too slow, these computed tables can easilky be moved to be compiled in. void basisu_transcoder_init() { if (g_transcoder_initialized) { - BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Called more than once\n"); + BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Called more than once\n"); return; } - - BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Initializing (this is not an error)\n"); + + BASISU_DEVEL_ERROR("basisu_transcoder::basisu_transcoder_init: Initializing (this is not an error)\n"); #if BASISD_SUPPORT_UASTC uastc_init(); @@ -2041,7 +2125,8 @@ namespace basist #if BASISD_SUPPORT_UASTC_HDR // TODO: Examine this, optimize for startup time/mem utilization. - astc_helpers::init_tables(false); + // XUASTC LDR decompressors need the rank tables + astc_helpers::init_tables(); astc_hdr_core_init(); #endif @@ -2049,7 +2134,7 @@ namespace basist #if BASISD_SUPPORT_ASTC transcoder_init_astc(); #endif - + #if BASISD_WRITE_NEW_ASTC_TABLES create_etc1_to_astc_conversion_table_0_47(); create_etc1_to_astc_conversion_table_0_255(); @@ -2159,11 +2244,25 @@ namespace basist astc_6x6_hdr::init_quantize_tables(); fast_encode_bc6h_init(); #endif - + #if BASISD_SUPPORT_BC7_MODE5 bc7_mode_5_encoder::encode_bc7_mode5_init(); #endif +#if BASISD_SUPPORT_XUASTC + // TODO: XUASTC support macro + astc_ldr_t::init(); + + astc_ldr_t::init_transcoding_tables(); + + // Used by arith encoder/decoder + arith_fastbits_f32::init(); + + // Used by astc ldr transcoding + bc7f::init(); + etc1f::init(); +#endif + g_transcoder_initialized = true; } @@ -2315,7 +2414,7 @@ namespace basist std::swap(l, h); pSelectors_xlat_256 = &g_etc1_to_dxt1_selector_mappings_raw_dxt1_inv_256[best_mapping][0]; } - + pDst_block->set_low_color(static_cast(l)); pDst_block->set_high_color(static_cast(h)); @@ -2475,7 +2574,7 @@ namespace basist fxt1_block* pBlock = static_cast(pDst); // CC_MIXED is basically DXT1 with different encoding tricks. - // So transcode ETC1S to DXT1, then transcode that to FXT1 which is easy and nearly lossless. + // So transcode ETC1S to DXT1, then transcode that to FXT1 which is easy and nearly lossless. // (It's not completely lossless because FXT1 rounds in its color lerps while DXT1 doesn't, but it should be good enough.) dxt1_block blk; convert_etc1s_to_dxt1(&blk, pEndpoints, pSelectors, false); @@ -2488,7 +2587,7 @@ namespace basist uint32_t g0 = color0.g & 1; uint32_t g1 = color1.g & 1; - + color0.g >>= 1; color1.g >>= 1; @@ -2496,7 +2595,7 @@ namespace basist blk.m_selectors[1] = conv_dxt1_to_fxt1_sels(blk.m_selectors[1]); blk.m_selectors[2] = conv_dxt1_to_fxt1_sels(blk.m_selectors[2]); blk.m_selectors[3] = conv_dxt1_to_fxt1_sels(blk.m_selectors[3]); - + if ((blk.get_selector(0, 0) >> 1) != (g0 ^ g1)) { std::swap(color0, color1); @@ -2510,7 +2609,7 @@ namespace basist if (fxt1_subblock == 0) { - pBlock->m_hi.m_mode = 1; + pBlock->m_hi.m_mode = 1; pBlock->m_hi.m_alpha = 0; pBlock->m_hi.m_glsb = g1 | (g1 << 1); pBlock->m_hi.m_r0 = color0.r; @@ -2831,7 +2930,7 @@ namespace basist { uint32_t r; decoder_etc_block::get_block_color5_r(base_color, inten_table, low_selector, r); - + pDst_block->set_low_alpha(r); pDst_block->set_high_alpha(r); pDst_block->m_selectors[0] = 0; @@ -2914,7 +3013,7 @@ namespace basist static const uint8_t g_pvrtc_4[16] = { 0,16,33,49,66,82,99,115,140,156,173,189,206,222,239,255 }; static const uint8_t g_pvrtc_3[8] = { 0,33,74,107,148,181,222,255 }; static const uint8_t g_pvrtc_alpha[9] = { 0,34,68,102,136,170,204,238,255 }; - + static const uint8_t g_pvrtc_5_floor[256] = { 0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3, @@ -2938,7 +3037,7 @@ namespace basist 24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,28, 28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31 }; - + static const uint8_t g_pvrtc_4_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, @@ -2962,7 +3061,7 @@ namespace basist 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14, 14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15 }; - + static const uint8_t g_pvrtc_3_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, @@ -2986,7 +3085,7 @@ namespace basist 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 }; - + static const uint8_t g_pvrtc_alpha_floor[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, @@ -3093,10 +3192,10 @@ namespace basist } assert((r < 32) && (g < 32) && (b < 32) && (a < 16)); - + return color32(r, g, b, a); } - + inline color32 get_endpoint_8888(uint32_t endpoint_index) const { assert(endpoint_index < 2); @@ -3143,7 +3242,7 @@ namespace basist a = g_pvrtc_alpha[a]; } - + return color32(r, g, b, a); } @@ -3152,7 +3251,7 @@ namespace basist color32 c(get_endpoint_8888(endpoint_index)); return c.r + c.g + c.b + c.a; } - + inline uint32_t get_opaque_endpoint_l0() const { uint32_t packed = m_endpoints & 0xFFFE; @@ -3267,7 +3366,7 @@ namespace basist else m_endpoints = (m_endpoints & 0xFFFF0000U) | packed; } - + // opaque endpoints: 554 or 555 // transparent endpoints: 3443 or 3444 inline void set_endpoint_raw(uint32_t endpoint_index, const color32& c, bool opaque_endpoint) @@ -3320,7 +3419,7 @@ namespace basist else m_endpoints = (m_endpoints & 0xFFFF0000U) | packed; } - + inline void set_endpoint_floor(uint32_t endpoint_index, const color32& c) { assert(endpoint_index < 2); @@ -3430,12 +3529,6 @@ namespace basist }; #endif - struct pvrtc1_temp_block - { - decoder_etc_block m_etc1_block; - uint32_t m_pvrtc_endpoints; - }; - static inline uint32_t get_opaque_endpoint_l0(uint32_t endpoints) { uint32_t packed = endpoints; @@ -3545,7 +3638,7 @@ namespace basist for (int ey = 0; ey < 3; ey++) { - int by = y + ey - 1; + int by = y + ey - 1; const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; @@ -3553,7 +3646,7 @@ namespace basist for (int ex = 0; ex < 3; ex++) { - int bx = 0 + ex - 1; + int bx = 0 + ex - 1; const uint32_t e = pE[bx & x_mask]; @@ -3702,8 +3795,8 @@ namespace basist } static void fixup_pvrtc1_4_modulation_rgba( - const decoder_etc_block* pETC_Blocks, - const uint32_t* pPVRTC_endpoints, + const decoder_etc_block* pETC_Blocks, + const uint32_t* pPVRTC_endpoints, void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, void *pAlpha_blocks, const endpoint* pEndpoints, const selector* pSelectors) { @@ -3726,7 +3819,7 @@ namespace basist for (int ey = 0; ey < 3; ey++) { - int by = y + ey - 1; + int by = y + ey - 1; const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; @@ -3734,7 +3827,7 @@ namespace basist for (int ex = 0; ex < 3; ex++) { - int bx = 0 + ex - 1; + int bx = 0 + ex - 1; const uint32_t e = pE[bx & x_mask]; @@ -3748,13 +3841,13 @@ namespace basist for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) { const decoder_etc_block& src_block = pETC_Blocks[block_index]; - + const uint16_t* pSrc_alpha_block = reinterpret_cast(static_cast(pAlpha_blocks) + x + (y * num_blocks_x)); const endpoint* pAlpha_endpoints = &pEndpoints[pSrc_alpha_block[0]]; const selector* pAlpha_selectors = &pSelectors[pSrc_alpha_block[1]]; - + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); - + uint32_t swizzled = x_swizzle | y_swizzle; if (num_blocks_x != num_blocks_y) { @@ -3897,7 +3990,7 @@ namespace basist const uint32_t NUM_ETC1_TO_BC7_M5_SELECTOR_RANGES = sizeof(g_etc1_to_bc7_m5_selector_ranges) / sizeof(g_etc1_to_bc7_m5_selector_ranges[0]); static uint32_t g_etc1_to_bc7_m5_selector_range_index[4][4]; - + const uint32_t NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS = 10; static const uint8_t g_etc1_to_bc7_m5_selector_mappings[NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS][4] = { @@ -3919,11 +4012,11 @@ namespace basist uint8_t m_hi; uint16_t m_err; }; - + static const etc1_to_bc7_m5_solution g_etc1_to_bc7_m5_color[32 * 8 * NUM_ETC1_TO_BC7_M5_SELECTOR_MAPPINGS * NUM_ETC1_TO_BC7_M5_SELECTOR_RANGES] = { #include "basisu_transcoder_tables_bc7_m5_color.inc" }; - + static dxt_selector_range g_etc1_to_bc7_m5a_selector_ranges[] = { { 0, 3 }, @@ -3948,7 +4041,7 @@ namespace basist { #include "basisu_transcoder_tables_bc7_m5_alpha.inc" }; - + static inline uint32_t set_block_bits(uint8_t* pBytes, uint32_t val, uint32_t num_bits, uint32_t cur_ofs) { assert(num_bits < 32); @@ -3973,83 +4066,6 @@ namespace basist return cur_ofs; } - struct bc7_mode_5 - { - union - { - struct - { - uint64_t m_mode : 6; - uint64_t m_rot : 2; - - uint64_t m_r0 : 7; - uint64_t m_r1 : 7; - uint64_t m_g0 : 7; - uint64_t m_g1 : 7; - uint64_t m_b0 : 7; - uint64_t m_b1 : 7; - uint64_t m_a0 : 8; - uint64_t m_a1_0 : 6; - - } m_lo; - - uint64_t m_lo_bits; - }; - - union - { - struct - { - uint64_t m_a1_1 : 2; - - // bit 2 - uint64_t m_c00 : 1; - uint64_t m_c10 : 2; - uint64_t m_c20 : 2; - uint64_t m_c30 : 2; - - uint64_t m_c01 : 2; - uint64_t m_c11 : 2; - uint64_t m_c21 : 2; - uint64_t m_c31 : 2; - - uint64_t m_c02 : 2; - uint64_t m_c12 : 2; - uint64_t m_c22 : 2; - uint64_t m_c32 : 2; - - uint64_t m_c03 : 2; - uint64_t m_c13 : 2; - uint64_t m_c23 : 2; - uint64_t m_c33 : 2; - - // bit 33 - uint64_t m_a00 : 1; - uint64_t m_a10 : 2; - uint64_t m_a20 : 2; - uint64_t m_a30 : 2; - - uint64_t m_a01 : 2; - uint64_t m_a11 : 2; - uint64_t m_a21 : 2; - uint64_t m_a31 : 2; - - uint64_t m_a02 : 2; - uint64_t m_a12 : 2; - uint64_t m_a22 : 2; - uint64_t m_a32 : 2; - - uint64_t m_a03 : 2; - uint64_t m_a13 : 2; - uint64_t m_a23 : 2; - uint64_t m_a33 : 2; - - } m_hi; - - uint64_t m_hi_bits; - }; - }; - #if BASISD_WRITE_NEW_BC7_MODE5_TABLES static void create_etc1_to_bc7_m5_color_conversion_table() { @@ -4095,7 +4111,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1_to_bc7_m5_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -4174,7 +4190,7 @@ namespace basist int mapping_err = block_colors[s].g - colors[k]; mapping_err *= mapping_err; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) mapping_err *= 5; @@ -4185,7 +4201,7 @@ namespace basist best_k = k; } } // k - + total_err += best_mapping_err; output_selectors |= (best_k << (s * 2)); } // s @@ -4200,7 +4216,7 @@ namespace basist } // lo } // hi - + fprintf(pFile, "{%u,%u,%u},", best_lo, best_hi, best_output_selectors); n++; if ((n & 31) == 31) @@ -4239,7 +4255,7 @@ namespace basist {127,104},{126,105},{126,106},{127,106},{127,107},{126,108},{125,109},{127,109},{126,110},{126,111},{127,111},{127,112},{126,113},{126,114},{127,114},{127,115}, {126,116},{126,117},{127,117},{127,118},{126,119},{126,120},{127,120},{127,121},{126,122},{126,123},{127,123},{127,124},{126,125},{126,126},{127,126},{127,127} }; - + static void transcoder_init_bc7_mode5() { #if 0 @@ -4267,9 +4283,9 @@ namespace basist } } // hi - + } // lo - + printf("{%u,%u},", g_bc7_m5_equals_1[i].m_hi, g_bc7_m5_equals_1[i].m_lo); if ((i & 15) == 15) printf("\n"); } @@ -4293,7 +4309,7 @@ namespace basist static void convert_etc1s_to_bc7_m5_color(void* pDst, const endpoint* pEndpoints, const selector* pSelector) { bc7_mode_5* pDst_block = static_cast(pDst); - + // First ensure the block is cleared to all 0's static_cast(pDst)[0] = 0; static_cast(pDst)[1] = 0; @@ -4419,7 +4435,7 @@ namespace basist pDst_block->m_lo.m_r1 = pTable_r[best_mapping].m_lo; pDst_block->m_lo.m_g1 = pTable_g[best_mapping].m_lo; pDst_block->m_lo.m_b1 = pTable_b[best_mapping].m_lo; - + s_inv = 3; } else @@ -4440,7 +4456,7 @@ namespace basist for (uint32_t x = 0; x < 4; x++) { const uint32_t s = pSelector->get_selector(x, y); - + const uint32_t os = pSelectors_xlat[s] ^ s_inv; output_bits |= (os << output_bit_ofs); @@ -4470,7 +4486,7 @@ namespace basist pDst_block->m_lo.m_a0 = r; pDst_block->m_lo.m_a1_0 = r & 63; pDst_block->m_hi.m_a1_1 = r >> 6; - + return; } else if (pSelector->m_num_unique_selectors == 2) @@ -4520,7 +4536,7 @@ namespace basist } const uint32_t selector_range_table = g_etc1_to_bc7_m5a_selector_range_index[low_selector][high_selector]; - + const etc1_g_to_bc7_m5a_conversion* pTable = &g_etc1_g_to_bc7_m5a[inten_table * (32 * NUM_ETC1_TO_BC7_M5A_SELECTOR_RANGES) + base_color_r * NUM_ETC1_TO_BC7_M5A_SELECTOR_RANGES + selector_range_table]; pDst_block->m_lo.m_a0 = pTable->m_lo; @@ -4561,10 +4577,12 @@ namespace basist set_block_bits((uint8_t*)pDst, output_bits, 31, 97); } +#if 0 static inline vec3F rgb_to_ycocg(const vec3F& rgb) { return vec3F(rgb.dot(vec3F(0.25f, 0.5f, 0.25f)), rgb.dot(vec3F(0.5f, 0.0f, -0.5f)), rgb.dot(vec3F(-0.25f, 0.5f, -0.25f))); } +#endif static inline vec2F rgb_to_cocg(const vec3F& rgb) { @@ -4576,11 +4594,14 @@ namespace basist return vec3F(ycocg.dot(vec3F(1.0f, 1.0f, -1.0f)), ycocg.dot(vec3F(1.0f, 0.0f, 1.0f)), ycocg.dot(vec3F(1.0f, -1.0f, -1.0f))); } +#if 0 static inline vec3F color32_to_vec3F(const color32& c) { return vec3F(c.r, c.g, c.b); } +#endif +#if 0 static inline vec3F color5_to_ycocg(const endpoint& e) { const int r = (e.m_color5[0] << 3) | (e.m_color5[0] >> 2); @@ -4588,6 +4609,7 @@ namespace basist const int b = (e.m_color5[2] << 3) | (e.m_color5[2] >> 2); return rgb_to_ycocg(vec3F((float)r, (float)g, (float)b)); } +#endif static inline vec2F color5_to_cocg(const endpoint& e) { @@ -4620,7 +4642,7 @@ namespace basist const bool hq_bc7_mode_5_encoder_mode = false; const int CHROMA_THRESH = 10; - + uint32_t total_filtered_blocks = 0; BASISU_NOTE_UNUSED(total_filtered_blocks); @@ -4629,7 +4651,7 @@ namespace basist for (int bx = 0; bx < (int)num_blocks_x; bx++) { vec2F center_cocg(color5_to_cocg(pEndpoints[decoded_endpoints(bx, by)])); - + //bool filter_flag = false; for (int dy = -1; dy <= 1; dy++) { @@ -4667,7 +4689,7 @@ namespace basist total_filtered_blocks++; bc7_mode_5* pDst_block = (bc7_mode_5*)(static_cast(pDst_blocks) + (bx + by * output_row_pitch_in_blocks_or_pixels) * sizeof(bc7_mode_5)); - + //memset(pDst_block, 0x80, 16); int lr = bc7_7_to_8(pDst_block->m_lo.m_r0); @@ -4691,7 +4713,7 @@ namespace basist float block_y_vals[16]; // [y][x] float y_sum = 0.0f, y_sum_sq = 0.0f; - + for (uint32_t i = 0; i < 16; i++) { uint32_t sel = sel_bits & (i ? 3 : 1); @@ -4700,7 +4722,7 @@ namespace basist block_y_vals[i] = y; y_sum += y; y_sum_sq += y * y; - + } // i const float S = 1.0f / 16.0f; @@ -4723,14 +4745,14 @@ namespace basist const float fy = ((float)((bpy + 2) & 3) + .5f) * (1.0f / 4.0f); const int ubx = bx + ((bpx - 2) >> 2); - + vec2F a(get_endpoint_cocg_clamped(ubx, uby, decoded_endpoints, pEndpoints)); vec2F b(get_endpoint_cocg_clamped(ubx + 1, uby, decoded_endpoints, pEndpoints)); vec2F c(get_endpoint_cocg_clamped(ubx, uby + 1, decoded_endpoints, pEndpoints)); vec2F d(get_endpoint_cocg_clamped(ubx + 1, uby + 1, decoded_endpoints, pEndpoints)); assert((fx >= 0) && (fx <= 1.0f) && (fy >= 0) && (fy <= 1.0f)); - + // TODO: Could merge this into 4 muls on each corner by weights vec2F ab = vec2F::lerp(a, b, fx); vec2F cd = vec2F::lerp(c, d, fx); @@ -4747,7 +4769,7 @@ namespace basist } // y bc7_mode_5_encoder::encode_bc7_mode_5_block(pDst_block, block_to_pack, hq_bc7_mode_5_encoder_mode); - + } // bx } // by @@ -5136,7 +5158,7 @@ namespace basist } #endif // BASISD_SUPPORT_ETC2_EAC_RG11 -// ASTC + // ASTC struct etc1_to_astc_solution { uint8_t m_lo; @@ -5183,7 +5205,7 @@ namespace basist // The best selector mapping to use given a base base+inten table and used selector range for converting grayscale data. static uint8_t g_etc1_to_astc_best_grayscale_mapping[32][8][NUM_ETC1_TO_ASTC_SELECTOR_RANGES]; - + #if BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY static const etc1_to_astc_solution g_etc1_to_astc_0_255[32 * 8 * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS * NUM_ETC1_TO_ASTC_SELECTOR_RANGES] = { #include "basisu_transcoder_tables_astc_0_255.inc" @@ -5248,7 +5270,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1_to_astc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 8; @@ -5269,7 +5291,7 @@ namespace basist mapping_best_high[m] = best_hi; mapping_best_err[m] = best_err; highest_best_err = basisu::maximum(highest_best_err, best_err); - + } // m for (uint32_t m = 0; m < NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS; m++) @@ -5345,7 +5367,7 @@ namespace basist { int err = block_colors[s].g - colors[g_etc1_to_astc_selector_mappings[m][s]]; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. int err_scale = 1; if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) @@ -5374,9 +5396,9 @@ namespace basist uint64_t err = mapping_best_err[m]; err = basisu::minimum(err, 0xFFFF); - + fprintf(pFile, "{%u,%u,%u},", mapping_best_low[m], mapping_best_high[m], (uint32_t)err); - + n++; if ((n & 31) == 31) fprintf(pFile, "\n"); @@ -5459,14 +5481,14 @@ namespace basist struct astc_block_params { // 2 groups of 5, but only a max of 8 are used (RRGGBBAA00) - uint8_t m_endpoints[10]; + uint8_t m_endpoints[10]; uint8_t m_weights[32]; }; - - // Packs a single format ASTC block using Color Endpoint Mode 12 (LDR RGBA direct), endpoint BISE range 13, 2-bit weights (range 2). + + // Packs a single format ASTC block using Color Endpoint Mode 12 (LDR RGBA direct), endpoint BISE range 13, 2-bit weights (range 2). // We're always going to output blocks containing alpha, even if the input doesn't have alpha, for simplicity. // Each block always has 4x4 weights, uses range 13 BISE encoding on the endpoints (0-47), and each weight ranges from 0-3. This encoding should be roughly equal in quality vs. BC1 for color. - // 8 total endpoints, stored as RGBA LH LH LH LH order, each ranging from 0-47. + // 8 total endpoints, stored as RGBA LH LH LH LH order, each ranging from 0-47. // Note the input [0,47] endpoint values are not linear - they are encoded as outlined in the ASTC spec: // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#astc-endpoint-unquantization // 32 total weights, stored as 16 CA CA, each ranging from 0-3. @@ -5488,7 +5510,7 @@ namespace basist astc_encode_trits(pOutput, pBlock->m_endpoints + 5, bit_pos, 4); // Pack 32 2-bit weights, which are stored from the top down into the block in opposite bit order. - + for (uint32_t i = 0; i < 32; i++) { static const uint8_t s_reverse_bits[4] = { 0, 2, 1, 3 }; @@ -5497,7 +5519,7 @@ namespace basist } } - // CEM mode 12 (LDR RGBA Direct), 8-bit endpoints, 1-bit weights + // CEM mode 12 (LDR RGBA Direct), 8-bit endpoints, 1-bit weights // This ASTC mode is basically block truncation coding (BTC) using 1-bit weights and 8-bit/component endpoints - very convenient. static void astc_pack_block_cem_12_weight_range0(uint32_t* pOutput, const astc_block_params* pBlock) { @@ -5535,7 +5557,7 @@ namespace basist // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#_block_mode pBytes[0] = 0x42; pBytes[1] = 0x84; pBytes[2] = 0x00; pBytes[3] = 0x00; pBytes[4] = 0x00; pBytes[5] = 0x00; pBytes[6] = 0x00; pBytes[7] = 0xc0; - + pOutput[2] = 0; pOutput[3] = 0; @@ -5561,7 +5583,7 @@ namespace basist // Write constant block mode, color component selector, number of partitions, color endpoint mode // https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#_block_mode pBytes[0] = 0x42; pBytes[1] = 0x00; pBytes[2] = 0x01; pBytes[3] = 0x00; - + pOutput[1] = 0; pOutput[2] = 0; pOutput[3] = 0; @@ -5589,7 +5611,7 @@ namespace basist { uint8_t m_lo, m_hi; } g_astc_single_color_encoding_1[256]; - + static void transcoder_init_astc() { for (uint32_t base_color = 0; base_color < 32; base_color++) @@ -5667,7 +5689,7 @@ namespace basist g_ise_to_unquant[bit | (trit << 4)] = unq; } } - + // Compute table used for optimal single color encoding. for (int i = 0; i < 256; i++) { @@ -5682,9 +5704,9 @@ namespace basist int l = lo_v | (lo_v << 8); int h = hi_v | (hi_v << 8); - + int v = ((l * (64 - 21) + (h * 21) + 32) / 64) >> 8; - + int e = abs(v - i); if (e < lowest_e) @@ -5706,7 +5728,7 @@ namespace basist for (int lo = 0; lo < 48; lo++) { const int lo_v = g_ise_to_unquant[lo]; - + int e = abs(lo_v - i); if (e < lowest_e) @@ -5721,7 +5743,7 @@ namespace basist // Converts opaque or color+alpha ETC1S block to ASTC 4x4. // This function tries to use the best ASTC mode given the block's actual contents. - static void convert_etc1s_to_astc_4x4(void* pDst_block, const endpoint* pEndpoints, const selector* pSelector, + static void convert_etc1s_to_astc_4x4(void* pDst_block, const endpoint* pEndpoints, const selector* pSelector, bool transcode_alpha, const endpoint *pEndpoint_codebook, const selector *pSelector_codebook) { astc_block_params blk; @@ -5765,7 +5787,7 @@ namespace basist // See https://www.khronos.org/registry/DataFormat/specs/1.2/dataformat.1.2.html#astc-void-extent-blocks uint32_t r, g, b; decoder_etc_block::get_block_color5(base_color, inten_table, low_selector, r, g, b); - + uint32_t* pOutput = static_cast(pDst_block); uint8_t* pBytes = reinterpret_cast(pDst_block); @@ -5785,7 +5807,7 @@ namespace basist } else if ((pSelector->m_num_unique_selectors <= 2) && (num_unique_alpha_selectors <= 2)) { - // Both color and alpha use <= 2 unique selectors each. + // Both color and alpha use <= 2 unique selectors each. // Use block truncation coding, which is lossless with ASTC (8-bit endpoints, 1-bit weights). color32 block_colors[4]; decoder_etc_block::get_block_colors5(block_colors, base_color, inten_table); @@ -5832,7 +5854,7 @@ namespace basist { uint32_t s = alpha_selectors.get_selector(x, y); s = (s == alpha_high_selector) ? 1 : 0; - + blk.m_weights[(x + y * 4) * 2 + 1] = static_cast(s); } // x } // y @@ -5865,12 +5887,12 @@ namespace basist return; } - + // Either alpha and/or color use > 2 unique selectors each, so we must do something more complex. - + #if BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY // The optional higher quality modes use 8-bits endpoints vs. [0,47] endpoints. - + // If the block's base color is grayscale, all pixels are grayscale, so encode the block as Luminance+Alpha. if ((base_color.r == base_color.g) && (base_color.r == base_color.b)) { @@ -5904,7 +5926,7 @@ namespace basist { // Convert ETC1S alpha const uint32_t alpha_selector_range_table = g_etc1_to_astc_selector_range_index[alpha_low_selector][alpha_high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc_0_255[(alpha_inten_table * 32 + alpha_base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + alpha_selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; @@ -5912,7 +5934,7 @@ namespace basist blk.m_endpoints[2] = pTable_g[best_mapping].m_lo; blk.m_endpoints[3] = pTable_g[best_mapping].m_hi; - + const uint8_t* pSelectors_xlat = &g_etc1_to_astc_selector_mappings[best_mapping][0]; for (uint32_t y = 0; y < 4; y++) @@ -5956,10 +5978,10 @@ namespace basist { // Convert ETC1S alpha const uint32_t selector_range_table = g_etc1_to_astc_selector_range_index[low_selector][high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc_0_255[(inten_table * 32 + base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; - + const uint32_t best_mapping = g_etc1_to_astc_best_grayscale_mapping_0_255[base_color.g][inten_table][selector_range_table]; blk.m_endpoints[0] = pTable_g[best_mapping].m_lo; @@ -6101,7 +6123,7 @@ namespace basist { // Convert ETC1S alpha const uint32_t alpha_selector_range_table = g_etc1_to_astc_selector_range_index[alpha_low_selector][alpha_high_selector]; - + //[32][8][RANGES][MAPPING] const etc1_to_astc_solution* pTable_g = &g_etc1_to_astc[(alpha_inten_table * 32 + alpha_base_color.g) * (NUM_ETC1_TO_ASTC_SELECTOR_RANGES * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS) + alpha_selector_range_table * NUM_ETC1_TO_ASTC_SELECTOR_MAPPINGS]; @@ -6145,7 +6167,7 @@ namespace basist const uint32_t r = block_colors[low_selector].r; const uint32_t g = block_colors[low_selector].g; const uint32_t b = block_colors[low_selector].b; - + blk.m_endpoints[0] = g_astc_single_color_encoding_1[r].m_lo; blk.m_endpoints[1] = g_astc_single_color_encoding_1[r].m_hi; @@ -6247,7 +6269,7 @@ namespace basist blk.m_endpoints[4] = pTable_b[best_mapping].m_lo; blk.m_endpoints[5] = pTable_b[best_mapping].m_hi; - + int s0 = g_ise_to_unquant[blk.m_endpoints[0]] + g_ise_to_unquant[blk.m_endpoints[2]] + g_ise_to_unquant[blk.m_endpoints[4]]; int s1 = g_ise_to_unquant[blk.m_endpoints[1]] + g_ise_to_unquant[blk.m_endpoints[3]] + g_ise_to_unquant[blk.m_endpoints[5]]; bool invert = false; @@ -6412,8 +6434,8 @@ namespace basist static void transcoder_init_atc() { prepare_atc_single_color_table(g_pvrtc2_match45_equals_1, 16, 32, 1); - prepare_atc_single_color_table(g_atc_match55_equals_1, 32, 32, 1); - prepare_atc_single_color_table(g_atc_match56_equals_1, 32, 64, 1); + prepare_atc_single_color_table(g_atc_match55_equals_1, 32, 32, 1); + prepare_atc_single_color_table(g_atc_match56_equals_1, 32, 64, 1); prepare_atc_single_color_table(g_pvrtc2_match4, 1, 16, 3); prepare_atc_single_color_table(g_atc_match5, 1, 32, 3); @@ -6467,7 +6489,7 @@ namespace basist pBlock->set_low_color(g_atc_match55_equals_1[r].m_lo, g_atc_match56_equals_1[g].m_lo, g_atc_match55_equals_1[b].m_lo); pBlock->set_high_color(g_atc_match55_equals_1[r].m_hi, g_atc_match56_equals_1[g].m_hi, g_atc_match55_equals_1[b].m_hi); - + pBlock->m_sels[0] = 0x55; pBlock->m_sels[1] = 0x55; pBlock->m_sels[2] = 0x55; @@ -6602,7 +6624,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6676,7 +6698,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6706,7 +6728,7 @@ namespace basist } // inten fclose(pFile); - + // PVRTC2 45 fopen_s(&pFile, "basisu_transcoder_tables_pvrtc2_45.inc", "w"); @@ -6751,7 +6773,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6828,7 +6850,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6905,7 +6927,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -6982,7 +7004,7 @@ namespace basist int err = block_colors[s].g - colors[g_etc1s_to_atc_selector_mappings[m][s]]; int err_scale = 1; - // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor + // Special case when the intensity table is 7, low_selector is 0, and high_selector is 3. In this extreme case, it's likely the encoder is trying to strongly favor // the low/high selectors which are clamping to either 0 or 255. if (((inten == 7) && (low_selector == 0) && (high_selector == 3)) && ((s == 0) || (s == 3))) err_scale = 5; @@ -7110,12 +7132,12 @@ namespace basist { uint8_t m_l, m_h; } g_pvrtc2_trans_match44[256]; - + static struct { uint8_t m_l, m_h; } g_pvrtc2_alpha_match33[256]; - + static struct { uint8_t m_l, m_h; @@ -7125,7 +7147,7 @@ namespace basist { uint8_t m_l, m_h; } g_pvrtc2_alpha_match33_3[256]; - + // PVRTC2 can be forced to look like a slightly weaker variant of ATC/BC1, so that's what we do here for simplicity. static void convert_etc1s_to_pvrtc2_rgb(void* pDst, const endpoint* pEndpoints, const selector* pSelector) { @@ -7237,7 +7259,7 @@ namespace basist pBlock->m_modulation[3] = (uint8_t)sels3; } } - + static inline vec4F* vec4F_set_scalar(vec4F* pV, float x) { pV->c[0] = x; pV->c[1] = x; pV->c[2] = x; pV->c[3] = x; return pV; } static inline vec4F* vec4F_set(vec4F* pV, float x, float y, float z, float w) { pV->c[0] = x; pV->c[1] = y; pV->c[2] = z; pV->c[3] = w; return pV; } static inline vec4F* vec4F_saturate_in_place(vec4F* pV) { pV->c[0] = saturate(pV->c[0]); pV->c[1] = saturate(pV->c[1]); pV->c[2] = saturate(pV->c[2]); pV->c[3] = saturate(pV->c[3]); return pV; } @@ -7255,9 +7277,9 @@ namespace basist } static inline int sq(int x) { return x * x; } - - // PVRTC2 is a slightly borked format for alpha: In Non-Interpolated mode, the way AlphaB8 is expanded from 4 to 8 bits means it can never be 0. - // This is actually very bad, because on 100% transparent blocks which have non-trivial color pixels, part of the color channel will leak into alpha! + + // PVRTC2 is a slightly borked format for alpha: In Non-Interpolated mode, the way AlphaB8 is expanded from 4 to 8 bits means it can never be 0. + // This is actually very bad, because on 100% transparent blocks which have non-trivial color pixels, part of the color channel will leak into alpha! // And there's nothing straightforward we can do because using the other modes is too expensive/complex. I can see why Apple didn't adopt it. static void convert_etc1s_to_pvrtc2_rgba(void* pDst, const endpoint* pEndpoints, const selector* pSelector, const endpoint* pEndpoint_codebook, const selector* pSelector_codebook) { @@ -7312,13 +7334,13 @@ namespace basist const uint32_t high_selector = pSelector->m_hi_selector; const int num_unique_color_selectors = pSelector->m_num_unique_selectors; - + // We need to reencode the block at the pixel level, unfortunately, from two ETC1S planes. // Do 4D incremental PCA, project all pixels to this hyperline, then quantize to packed endpoints and compute the modulation values. const int br = (base_color.r << 3) | (base_color.r >> 2); const int bg = (base_color.g << 3) | (base_color.g >> 2); const int bb = (base_color.b << 3) | (base_color.b >> 2); - + color32 block_cols[4]; for (uint32_t i = 0; i < 4; i++) { @@ -7347,14 +7369,14 @@ namespace basist decoder_etc_block::get_block_color5(base_color, inten_table, low_selector, r, g, b); // Mod 0 - uint32_t lr0 = (r * 15 + 128) / 255, lg0 = (g * 15 + 128) / 255, lb0 = (b * 7 + 128) / 255; + uint32_t lr0 = (r * 15 + 128) / 255, lg0 = (g * 15 + 128) / 255, lb0 = (b * 7 + 128) / 255; uint32_t la0 = g_pvrtc2_alpha_match33_0[constant_alpha_val].m_l; uint32_t cr0 = (lr0 << 1) | (lr0 >> 3); uint32_t cg0 = (lg0 << 1) | (lg0 >> 3); uint32_t cb0 = (lb0 << 2) | (lb0 >> 1); uint32_t ca0 = (la0 << 1); - + cr0 = (cr0 << 3) | (cr0 >> 2); cg0 = (cg0 << 3) | (cg0 >> 2); cb0 = (cb0 << 3) | (cb0 >> 2); @@ -7383,14 +7405,14 @@ namespace basist uint32_t cg3 = (lg3 << 1) | (lg3 >> 3); uint32_t cb3 = (lb3 << 1) | (lb3 >> 3); uint32_t ca3 = (la3 << 1) | 1; - + cr3 = (cr3 << 3) | (cr3 >> 2); cg3 = (cg3 << 3) | (cg3 >> 2); cb3 = (cb3 << 3) | (cb3 >> 2); ca3 = (ca3 << 4) | ca3; uint32_t err3 = sq(cr3 - r) + sq(cg3 - g) + sq(cb3 - b) + sq(ca3 - constant_alpha_val) * 2; - + // Mod 1 uint32_t lr1 = g_pvrtc2_trans_match44[r].m_l, lg1 = g_pvrtc2_trans_match44[g].m_l, lb1 = g_pvrtc2_trans_match34[b].m_l; uint32_t hr1 = g_pvrtc2_trans_match44[r].m_h, hg1 = g_pvrtc2_trans_match44[g].m_h, hb1 = g_pvrtc2_trans_match34[b].m_h; @@ -7465,7 +7487,7 @@ namespace basist // It's a solid color block. uint32_t low_a = block_cols[alpha_selectors.m_lo_selector].a; uint32_t high_a = block_cols[alpha_selectors.m_hi_selector].a; - + const float S = 1.0f / 255.0f; vec4F_set(&minColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, low_a * S); vec4F_set(&maxColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, high_a * S); @@ -7477,7 +7499,7 @@ namespace basist vec4F_set(&minColor, block_cols[low_selector].r * S, block_cols[low_selector].g * S, block_cols[low_selector].b * S, constant_alpha_val * S); vec4F_set(&maxColor, block_cols[high_selector].r * S, block_cols[high_selector].g * S, block_cols[high_selector].b * S, constant_alpha_val * S); } - // See if any of the block colors got clamped - if so the principle axis got distorted (it's no longer just the ETC1S luma axis). + // See if any of the block colors got clamped - if so the principle axis got distorted (it's no longer just the ETC1S luma axis). // To keep quality up we need to use full 4D PCA in this case. else if ((block_cols[low_selector].c[0] == 0) || (block_cols[high_selector].c[0] == 255) || (block_cols[low_selector].c[1] == 0) || (block_cols[high_selector].c[1] == 255) || @@ -7528,7 +7550,7 @@ namespace basist } vec4F_normalize_in_place(&axis); - + if (vec4F_dot(&axis, &axis) < .5f) vec4F_set_scalar(&axis, .5f); @@ -7628,10 +7650,10 @@ namespace basist // 4433 4443 color32 trialMinColor, trialMaxColor; - + trialMinColor.set_clamped((int)(minColor.c[0] * 15.0f + .5f), (int)(minColor.c[1] * 15.0f + .5f), (int)(minColor.c[2] * 7.0f + .5f), (int)(minColor.c[3] * 7.0f + .5f)); trialMaxColor.set_clamped((int)(maxColor.c[0] * 15.0f + .5f), (int)(maxColor.c[1] * 15.0f + .5f), (int)(maxColor.c[2] * 15.0f + .5f), (int)(maxColor.c[3] * 7.0f + .5f)); - + pBlock->set_trans_low_color(trialMinColor.r, trialMinColor.g, trialMinColor.b, trialMinColor.a); pBlock->set_trans_high_color(trialMaxColor.r, trialMaxColor.g, trialMaxColor.b, trialMaxColor.a); @@ -7704,7 +7726,7 @@ namespace basist } } } - + static void transcoder_init_pvrtc2() { for (uint32_t v = 0; v < 256; v++) @@ -7810,7 +7832,7 @@ namespace basist g_pvrtc2_trans_match34[v].m_l = (uint8_t)best_l; g_pvrtc2_trans_match34[v].m_h = (uint8_t)best_h; } - + for (uint32_t v = 0; v < 256; v++) { int best_l = 0, best_h = 0, lowest_err = INT_MAX; @@ -7844,12 +7866,12 @@ namespace basist #endif // BASISD_SUPPORT_PVRTC2 //------------------------------------------------------------------------------------------------ - + // BC7 mode 5 RGB encoder #if BASISD_SUPPORT_BC7_MODE5 namespace bc7_mode_5_encoder - { + { static float g_mode5_rgba_midpoints[128]; void encode_bc7_mode5_init() @@ -8124,10 +8146,10 @@ namespace basist } int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 - + // TODO: Tune this const int32_t SIMPLE_BLOCK_THRESH = 10 * 16; - + if ((!hq_mode) && (block_max_var < SIMPLE_BLOCK_THRESH)) { const int L = 16, H = 239; @@ -8168,7 +8190,7 @@ namespace basist saxis_g = (int)(alt_xg * m); saxis_b = (int)(alt_xb * m); } - + saxis_r = (int)((uint32_t)saxis_r << 4U); saxis_g = (int)((uint32_t)saxis_g << 4U); saxis_b = (int)((uint32_t)saxis_b << 4U); @@ -8320,7 +8342,7 @@ namespace basist sym_codec.stop(); m_local_selectors.resize(num_selectors); - + if (!sym_codec.init(pSelectors_data, selectors_data_size)) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::decode_palettes: fail 5\n"); @@ -8345,7 +8367,7 @@ namespace basist BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::decode_palettes: hybrid global selector codebooks are unsupported\n"); return false; } - + const bool used_raw_encoding = (sym_codec.get_bits(1) == 1); if (used_raw_encoding) @@ -8526,7 +8548,7 @@ namespace basist if (!output_rows_in_pixels) output_rows_in_pixels = orig_height; } - + basisu::vector* pPrev_frame_indices = nullptr; if (is_video) { @@ -8554,12 +8576,12 @@ namespace basist } approx_move_to_front selector_history_buf(m_selector_history_buf_size); - + uint32_t cur_selector_rle_count = 0; decoder_etc_block block; memset(&block, 0, sizeof(block)); - + //block.set_flip_bit(true); // Setting the flip bit to false to be compatible with the Khronos KDFS. block.set_flip_bit(false); @@ -8595,7 +8617,7 @@ namespace basist if (!endpoints.size() || !selectors.size()) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: global codebooks must be unpacked first\n"); - + if (pPVRTC_work_mem) free(pPVRTC_work_mem); @@ -8606,7 +8628,7 @@ namespace basist const uint32_t SELECTOR_HISTORY_BUF_RLE_SYMBOL_INDEX = m_selector_history_buf_size + SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX; #if BASISD_SUPPORT_BC7_MODE5 - const bool bc7_chroma_filtering = ((decode_flags & cDecodeFlagsNoETC1SChromaFiltering) == 0) && + const bool bc7_chroma_filtering = ((decode_flags & cDecodeFlagsNoETC1SChromaFiltering) == 0) && ((fmt == block_format::cBC7_M5_COLOR) || (fmt == block_format::cBC7)); basisu::vector2D decoded_endpoints; @@ -8831,7 +8853,7 @@ namespace basist case block_format::cETC1: { decoder_etc_block* pDst_block = reinterpret_cast(static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes); - + block.set_base5_color(decoder_etc_block::pack_color5(pEndpoints->m_color5, false)); block.set_inten_table(0, pEndpoints->m_inten5); block.set_inten_table(1, pEndpoints->m_inten5); @@ -8882,7 +8904,7 @@ namespace basist const uint32_t low_selector = pSelector->m_lo_selector; const uint32_t high_selector = pSelector->m_hi_selector; - // Get block's RGB bounding box + // Get block's RGB bounding box color32 block_colors[2]; decoder_etc_block::get_block_colors5_bounds(block_colors, base_color, inten_table, low_selector, high_selector); @@ -8898,7 +8920,7 @@ namespace basist pPVRTC_endpoints[block_x + block_y * num_blocks_x] = temp.m_endpoints; #else assert(0); -#endif +#endif break; } @@ -8906,7 +8928,7 @@ namespace basist { #if BASISD_SUPPORT_PVRTC1 assert(pAlpha_blocks); - + block.set_base5_color(decoder_etc_block::pack_color5(pEndpoints->m_color5, false)); block.set_inten_table(0, pEndpoints->m_inten5); block.set_inten_table(1, pEndpoints->m_inten5); @@ -8914,7 +8936,7 @@ namespace basist ((decoder_etc_block*)pPVRTC_work_mem)[block_x + block_y * num_blocks_x] = block; - // Get block's RGBA bounding box + // Get block's RGBA bounding box const color32& base_color = pEndpoints->m_color5; const uint32_t inten_table = pEndpoints->m_inten5; const uint32_t low_selector = pSelector->m_lo_selector; @@ -8949,7 +8971,7 @@ namespace basist pPVRTC_endpoints[block_x + block_y * num_blocks_x] = temp.m_endpoints; #else assert(0); -#endif +#endif break; } @@ -8990,7 +9012,7 @@ namespace basist #endif break; } - case block_format::cASTC_4x4: + case block_format::cASTC_LDR_4x4: { #if BASISD_SUPPORT_ASTC void* pDst_block = static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes; @@ -9039,7 +9061,7 @@ namespace basist assert(transcode_alpha); void* pDst_block = static_cast(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes; - + convert_etc1s_to_pvrtc2_rgba(pDst_block, pEndpoints, pSelector, &endpoints[0], &selectors[0]); #endif break; @@ -9055,10 +9077,10 @@ namespace basist { assert(sizeof(uint32_t) == output_block_or_pixel_stride_in_bytes); uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint32_t); - + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); - + int colors[4]; decoder_etc_block::get_block_colors5_g(colors, pEndpoints->m_color5, pEndpoints->m_inten5); @@ -9072,7 +9094,7 @@ namespace basist pDst_pixels[3+4] = static_cast(colors[(s >> 2) & 3]); pDst_pixels[3+8] = static_cast(colors[(s >> 4) & 3]); pDst_pixels[3+12] = static_cast(colors[(s >> 6) & 3]); - + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t); } } @@ -9101,7 +9123,7 @@ namespace basist color32 colors[4]; decoder_etc_block::get_block_colors5(colors, pEndpoints->m_color5, pEndpoints->m_inten5); - + for (uint32_t y = 0; y < max_y; y++) { const uint32_t s = pSelector->m_selectors[y]; @@ -9222,7 +9244,7 @@ namespace basist cur = byteswap_uint16(cur); cur = (cur & 0xF) | packed_colors[(s >> (x * 2)) & 3]; - + if (BASISD_IS_BIG_ENDIAN) cur = byteswap_uint16(cur); @@ -9322,7 +9344,7 @@ namespace basist if (endpoint_pred_repeat_count != 0) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: endpoint_pred_repeat_count != 0. The file is corrupted or this is a bug\n"); - + if (pPVRTC_work_mem) free(pPVRTC_work_mem); @@ -9353,22 +9375,19 @@ namespace basist } bool basis_validate_output_buffer_size( - basis_tex_format source_format, transcoder_texture_format target_format, uint32_t output_blocks_buf_size_in_blocks_or_pixels, uint32_t orig_width, uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels) { - BASISU_NOTE_UNUSED(source_format); - if (basis_transcoder_format_is_uncompressed(target_format)) { // Assume the output buffer is orig_width by orig_height if (!output_row_pitch_in_blocks_or_pixels) output_row_pitch_in_blocks_or_pixels = orig_width; - if (!output_rows_in_pixels) + if (!output_rows_in_pixels) output_rows_in_pixels = orig_height; // Now make sure the output buffer is large enough, or we'll overwrite memory. @@ -9380,11 +9399,12 @@ namespace basist } else { + // Take into account the destination format's block width/height. const uint32_t dst_block_width = basis_get_block_width(target_format); const uint32_t dst_block_height = basis_get_block_height(target_format); //const uint32_t bytes_per_block = basis_get_bytes_per_block_or_pixel(target_format); - - // Take into account the destination format's block width/height. + + // Compute how many blocks should be in the output. const uint32_t num_dst_blocks_x = (orig_width + dst_block_width - 1) / dst_block_width; const uint32_t num_dst_blocks_y = (orig_height + dst_block_height - 1) / dst_block_height; const uint32_t total_dst_blocks = num_dst_blocks_x * num_dst_blocks_y; @@ -9402,7 +9422,7 @@ namespace basist return true; } - + uint32_t basis_compute_transcoded_image_size_in_bytes(transcoder_texture_format target_format, uint32_t orig_width, uint32_t orig_height) { assert(orig_width && orig_height); @@ -9418,7 +9438,7 @@ namespace basist const uint32_t bytes_per_slice = bytes_per_line * orig_height; return bytes_per_slice; } - + // Compressed formats are 2D arrays of blocks. const uint32_t bytes_per_block = basis_get_bytes_per_block_or_pixel(target_format); @@ -9489,12 +9509,12 @@ namespace basist // Switch to PVRTC1 RGB if the input doesn't have alpha. target_format = transcoder_texture_format::cTFPVRTC1_4_RGB; } - + const bool transcode_alpha_data_to_opaque_formats = (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - - if (!basis_validate_output_buffer_size(basis_tex_format::cETC1S, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -9520,7 +9540,7 @@ namespace basist { //status = transcode_slice(pData, data_size, slice_index_to_decode, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pData, data_len, block_format::cETC1, bytes_per_block_or_pixel, false, is_video, is_alpha_slice, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); - + if (!status) { BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); @@ -9645,7 +9665,7 @@ namespace basist if (basis_file_has_alpha_slices) { - // First decode the alpha data + // First decode the alpha data //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_A8, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cETC2_EAC_A8, bytes_per_block_or_pixel, false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); } @@ -9683,8 +9703,8 @@ namespace basist return false; #else assert(bytes_per_block_or_pixel == 16); - - // First decode the alpha data + + // First decode the alpha data if (basis_file_has_alpha_slices) { //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); @@ -9754,7 +9774,7 @@ namespace basist break; #endif } - case transcoder_texture_format::cTFASTC_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: { #if !BASISD_SUPPORT_ASTC BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: ASTC unsupported\n"); @@ -9771,13 +9791,13 @@ namespace basist { // Now decode the color data and transcode to ASTC. The transcoder function will read the alpha selector data from the output texture as it converts and // transcode both the alpha and color data at the same time to ASTC. - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, 16, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); + //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_LDR_4x4, 16, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); } } else - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); + //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_LDR_4x4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); if (!status) { @@ -9813,7 +9833,7 @@ namespace basist #else assert(bytes_per_block_or_pixel == 16); - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) { //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, 16, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); @@ -9873,7 +9893,7 @@ namespace basist } else { - // Now decode the color data and transcode to PVRTC2 RGBA. + // Now decode the color data and transcode to PVRTC2 RGBA. //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC2_4_RGBA, bytes_per_block_or_pixel, decode_flags | cDecodeFlagsOutputHasAlphaIndices, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + rgb_offset, rgb_length, block_format::cPVRTC2_4_RGBA, bytes_per_block_or_pixel, false, is_video, false, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, true, nullptr, output_rows_in_pixels, decode_flags); } @@ -9894,7 +9914,7 @@ namespace basist { // Raw 32bpp pixels, decoded in the usual raster order (NOT block order) into an image in memory. - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cA32, sizeof(uint32_t), decode_flags, output_row_pitch_in_blocks_or_pixels, pState, nullptr, output_rows_in_pixels); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cA32, sizeof(uint32_t), false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); @@ -9935,7 +9955,7 @@ namespace basist { // Raw 16bpp pixels, decoded in the usual raster order (NOT block order) into an image in memory. - // First decode the alpha data + // First decode the alpha data if (basis_file_has_alpha_slices) //status = transcode_slice(pData, data_size, slice_index + 1, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA4444_ALPHA, sizeof(uint16_t), decode_flags, output_row_pitch_in_blocks_or_pixels, pState, nullptr, output_rows_in_pixels); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + alpha_offset, alpha_length, block_format::cRGBA4444_ALPHA, sizeof(uint16_t), false, is_video, true, level_index, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, false, nullptr, output_rows_in_pixels, decode_flags); @@ -10039,14 +10059,15 @@ namespace basist } //------------------------------------------------------------------------------------------------ - + // UASTC LDR 4x4 transcoder + //------------------------------------------------------------------------------------------------ basisu_lowlevel_uastc_ldr_4x4_transcoder::basisu_lowlevel_uastc_ldr_4x4_transcoder() { } bool basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, - uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) { @@ -10107,7 +10128,7 @@ namespace basist for (uint32_t block_y = 0; block_y < num_blocks_y; ++block_y) { void* pDst_block = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; - + for (uint32_t block_x = 0; block_x < num_blocks_x; ++block_x, ++pSource_block, pDst_block = (uint8_t *)pDst_block + output_block_or_pixel_stride_in_bytes) { switch (fmt) @@ -10143,7 +10164,7 @@ namespace basist } case block_format::cBC4: { - if (channel0 < 0) + if (channel0 < 0) channel0 = 0; status = transcode_uastc_to_bc4(*pSource_block, pDst_block, high_quality, channel0); break; @@ -10163,7 +10184,7 @@ namespace basist status = transcode_uastc_to_bc7(*pSource_block, pDst_block); break; } - case block_format::cASTC_4x4: + case block_format::cASTC_LDR_4x4: { status = transcode_uastc_to_astc(*pSource_block, pDst_block); break; @@ -10306,7 +10327,7 @@ namespace basist return false; #endif } - + bool basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image( transcoder_texture_format target_format, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, @@ -10328,7 +10349,7 @@ namespace basist { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: source data buffer too small\n"); return false; - } + } if ((target_format == transcoder_texture_format::cTFPVRTC1_4_RGB) || (target_format == transcoder_texture_format::cTFPVRTC1_4_RGBA)) { @@ -10350,12 +10371,12 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cUASTC4x4, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: output buffer size too small\n"); return false; } - + bool status = false; // UASTC4x4 @@ -10363,10 +10384,9 @@ namespace basist { case transcoder_texture_format::cTFETC1_RGB: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC1, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); - + if (!status) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); @@ -10375,7 +10395,6 @@ namespace basist } case transcoder_texture_format::cTFETC2_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_RGBA, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_RGBA, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10387,7 +10406,6 @@ namespace basist case transcoder_texture_format::cTFBC1_RGB: { // TODO: ETC1S allows BC1 from alpha channel. That doesn't seem actually useful, though. - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC1, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC1, bytes_per_block_or_pixel, true, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10398,7 +10416,6 @@ namespace basist } case transcoder_texture_format::cTFBC3_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC3, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC3, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) @@ -10409,9 +10426,6 @@ namespace basist } case transcoder_texture_format::cTFBC4_R: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC4, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC4, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); @@ -10423,9 +10437,6 @@ namespace basist } case transcoder_texture_format::cTFBC5_RG: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC5, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // 0, 3); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC5, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, 0, 3, decode_flags); @@ -10438,7 +10449,6 @@ namespace basist case transcoder_texture_format::cTFBC7_RGBA: case transcoder_texture_format::cTFBC7_ALT: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBC7, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC7, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10449,7 +10459,6 @@ namespace basist } case transcoder_texture_format::cTFPVRTC1_4_RGB: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC1_4_RGB, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGB, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10460,7 +10469,6 @@ namespace basist } case transcoder_texture_format::cTFPVRTC1_4_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cPVRTC1_4_RGBA, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGBA, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10469,10 +10477,9 @@ namespace basist } break; } - case transcoder_texture_format::cTFASTC_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cASTC_4x4, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); - status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cASTC_4x4, + status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cASTC_LDR_4x4, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) { @@ -10483,29 +10490,26 @@ namespace basist case transcoder_texture_format::cTFATC_RGB: case transcoder_texture_format::cTFATC_RGBA: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->ATC currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->ATC currently unsupported\n"); return false; } case transcoder_texture_format::cTFFXT1_RGB: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->FXT1 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->FXT1 currently unsupported\n"); return false; } case transcoder_texture_format::cTFPVRTC2_4_RGB: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->PVRTC2 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); return false; } case transcoder_texture_format::cTFPVRTC2_4_RGBA: { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC->PVRTC2 currently unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_ldr_4x4_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); return false; } case transcoder_texture_format::cTFETC2_EAC_R11: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_R11, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_R11, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); @@ -10517,9 +10521,6 @@ namespace basist } case transcoder_texture_format::cTFETC2_EAC_RG11: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cETC2_EAC_RG11, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState, - // nullptr, 0, - // 0, 3); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_RG11, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, 0, 3, decode_flags); @@ -10531,7 +10532,6 @@ namespace basist } case transcoder_texture_format::cTFRGBA32: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA32, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA32, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10542,7 +10542,6 @@ namespace basist } case transcoder_texture_format::cTFRGB565: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGB565, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGB565, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10553,7 +10552,6 @@ namespace basist } case transcoder_texture_format::cTFBGR565: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cBGR565, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBGR565, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10564,7 +10562,6 @@ namespace basist } case transcoder_texture_format::cTFRGBA4444: { - //status = transcode_slice(pData, data_size, slice_index, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, block_format::cRGBA4444, bytes_per_block_or_pixel, decode_flags, output_row_pitch_in_blocks_or_pixels, pState); status = transcode_slice(pOutput_blocks, num_blocks_x, num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA4444, bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); if (!status) @@ -10583,9 +10580,10 @@ namespace basist return status; } - + + //------------------------------------------------------------------------------------------------ + // UASTC HDR 4x4 transcoding //------------------------------------------------------------------------------------------------ - // UASTC HDR 4x4 basisu_lowlevel_uastc_hdr_4x4_transcoder::basisu_lowlevel_uastc_hdr_4x4_transcoder() { @@ -10593,7 +10591,7 @@ namespace basist bool basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, - uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) { @@ -10642,7 +10640,7 @@ namespace basist bool status = false; // TODO: Optimize pure memcpy() case. - + for (uint32_t block_y = 0; block_y < num_blocks_y; ++block_y) { void* pDst_block = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; @@ -10677,7 +10675,7 @@ namespace basist uint32_t blk_texels[4][4]; status = astc_helpers::decode_block(log_blk, blk_texels, 4, 4, astc_helpers::cDecodeModeRGB9E5); - + if (status) { const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); @@ -10691,7 +10689,7 @@ namespace basist } // y } } - + break; } case block_format::cRGBA_HALF: @@ -10703,7 +10701,7 @@ namespace basist half_float* pDst_pixels = reinterpret_cast( static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(half_float) * 4 ); - + half_float blk_texels[4][4][4]; status = astc_helpers::decode_block(log_blk, blk_texels, 4, 4, astc_helpers::cDecodeModeHDR16); @@ -10769,7 +10767,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice: Transcoder failed to unpack a UASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_slice: Transcoder failed to unpack a UASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -10825,7 +10823,7 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cUASTC_HDR_4x4, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_4x4_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -10962,7 +10960,7 @@ namespace basist assert(((orig_width + 5) / 6) == num_blocks_x); assert(((orig_height + 5) / 6) == num_blocks_y); - + if (fmt == block_format::cBC6H) { const uint32_t num_dst_blocks_x = (orig_width + 3) / 4; @@ -10987,7 +10985,7 @@ namespace basist fast_bc6h_params bc6h_enc_params; const bool hq_flag = (decode_flags & cDecodeFlagsHighQuality) != 0; bc6h_enc_params.m_max_2subset_pats_to_try = hq_flag ? 1 : 0; - + for (uint32_t src_block_y = 0; src_block_y < num_blocks_y; src_block_y += 2) { const uint32_t num_inner_blocks_y = basisu::minimum(2, num_blocks_y - src_block_y); @@ -11003,7 +11001,7 @@ namespace basist const astc_blk* pS = pSource_block + (src_block_y + iy) * num_blocks_x + (src_block_x + ix); half_float blk_texels[6][6][4]; - + astc_helpers::log_astc_block log_blk; status = astc_helpers::unpack_block(pS, log_blk, 6, 6); if (!status) @@ -11011,7 +11009,7 @@ namespace basist BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } - + status = astc_helpers::decode_block(log_blk, blk_texels, 6, 6, astc_helpers::cDecodeModeHDR16); if (!status) { @@ -11026,14 +11024,14 @@ namespace basist unpacked_blocks[iy * 6 + y][ix * 6 + x][0] = blk_texels[y][x][0]; unpacked_blocks[iy * 6 + y][ix * 6 + x][1] = blk_texels[y][x][1]; unpacked_blocks[iy * 6 + y][ix * 6 + x][2] = blk_texels[y][x][2]; - + } // x } // y } // ix } // iy - + const uint32_t dst_x = src_block_x * 6; assert((dst_x & 3) == 0); const uint32_t dst_block_x = dst_x >> 2; @@ -11066,10 +11064,10 @@ namespace basist src_pixels[y][x][0] = unpacked_blocks[src_pixel_y][src_pixel_x][0]; src_pixels[y][x][1] = unpacked_blocks[src_pixel_y][src_pixel_x][1]; src_pixels[y][x][2] = unpacked_blocks[src_pixel_y][src_pixel_x][2]; - + } // x } // y - + astc_6x6_hdr::fast_encode_bc6h(&src_pixels[0][0][0], pDst_block, bc6h_enc_params); } // dx @@ -11078,7 +11076,7 @@ namespace basist } // block_x } // block_y - + status = true; } else @@ -11204,7 +11202,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11261,7 +11259,7 @@ namespace basist const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cASTC_HDR_6x6, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_transcoder::transcode_image: output buffer size too small\n"); return false; @@ -11334,14 +11332,14 @@ namespace basist } //------------------------------------------------------------------------------------------------ - // ASTC 6x6 HDR intermediate + // UASTC 6x6 HDR intermediate - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder() + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder() { } // num_blocks_x/num_blocks_y are source 6x6 blocks - bool basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice( + bool basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice( void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, @@ -11359,7 +11357,7 @@ namespace basist assert(g_transcoder_initialized); if (!g_transcoder_initialized) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder not globally initialized.\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder not globally initialized.\n"); return false; } @@ -11372,14 +11370,14 @@ namespace basist bool dec_status = astc_6x6_hdr::decode_6x6_hdr(pImage_data, image_data_size, decoded_blocks, dec_width, dec_height); if (!dec_status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: decode_6x6_hdr() failed.\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: decode_6x6_hdr() failed.\n"); return false; } if ((dec_width != orig_width) || (dec_height != orig_height) || (decoded_blocks.get_width() != num_blocks_x) || (decoded_blocks.get_height() != num_blocks_y)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: unexpected decoded width/height\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: unexpected decoded width/height\n"); return false; } @@ -11422,20 +11420,20 @@ namespace basist } else if (output_row_pitch_in_blocks_or_pixels < num_dst_blocks_x) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: output_row_pitch_in_blocks_or_pixels is too low\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: output_row_pitch_in_blocks_or_pixels is too low\n"); return false; } if (output_block_or_pixel_stride_in_bytes != sizeof(bc6h_block)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: invalid output_block_or_pixel_stride_in_bytes\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: invalid output_block_or_pixel_stride_in_bytes\n"); return false; } fast_bc6h_params bc6h_enc_params; const bool hq_flag = (decode_flags & cDecodeFlagsHighQuality) != 0; bc6h_enc_params.m_max_2subset_pats_to_try = hq_flag ? 1 : 0; - + for (uint32_t src_block_y = 0; src_block_y < num_blocks_y; src_block_y += 2) { const uint32_t num_inner_blocks_y = basisu::minimum(2, num_blocks_y - src_block_y); @@ -11456,14 +11454,14 @@ namespace basist status = astc_helpers::unpack_block(pS, log_blk, 6, 6); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } status = astc_helpers::decode_block(log_blk, blk_texels, 6, 6, astc_helpers::cDecodeModeHDR16); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11516,7 +11514,7 @@ namespace basist } // x } // y - + astc_6x6_hdr::fast_encode_bc6h(&src_pixels[0][0][0], pDst_block, bc6h_enc_params); } // dx @@ -11651,7 +11649,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: Transcoder failed to unpack a ASTC HDR block - this is a bug, or the data was corrupted\n"); return false; } @@ -11662,7 +11660,7 @@ namespace basist return true; #else - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_slice: ASTC HDR is unsupported\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_slice: ASTC HDR is unsupported\n"); BASISU_NOTE_UNUSED(decode_flags); BASISU_NOTE_UNUSED(channel0); @@ -11681,7 +11679,7 @@ namespace basist #endif } - bool basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image( + bool basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image( transcoder_texture_format target_format, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, const uint8_t* pCompressed_data, uint32_t compressed_data_length, @@ -11701,16 +11699,16 @@ namespace basist if (((uint64_t)slice_offset + slice_length) > (uint64_t)compressed_data_length) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: source data buffer too small\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: source data buffer too small\n"); return false; } const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); //const uint32_t total_slice_blocks = num_blocks_x * num_blocks_y; - if (!basis_validate_output_buffer_size(basis_tex_format::cASTC_HDR_6x6, target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: output buffer size too small\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: output buffer size too small\n"); return false; } @@ -11725,7 +11723,7 @@ namespace basist if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to ASTC_HDR failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to ASTC_HDR failed\n"); } break; } @@ -11735,7 +11733,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to BC6H failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to BC6H failed\n"); } break; } @@ -11745,7 +11743,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGB_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGB_HALF failed\n"); } break; } @@ -11755,7 +11753,7 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); } break; } @@ -11765,14 +11763,14 @@ namespace basist bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1 , decode_flags); if (!status) { - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: transcode_slice() to RGBA_HALF failed\n"); } break; } default: { assert(0); - BASISU_DEVEL_ERROR("basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder::transcode_image: Invalid format\n"); + BASISU_DEVEL_ERROR("basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder::transcode_image: Invalid format\n"); break; } } @@ -11781,7 +11779,7 @@ namespace basist } //------------------------------------------------------------------------------------------------ - + basisu_transcoder::basisu_transcoder() : m_ready_to_transcode(false) { @@ -11809,7 +11807,7 @@ namespace basist return false; } } -#endif +#endif return true; } @@ -11896,7 +11894,7 @@ namespace basist return false; } } - + // This flag dates back to pre-Basis Universal, when .basis supported full ETC1 too. if ((pHeader->m_flags & cBASISHeaderFlagETC1S) == 0) { @@ -11912,7 +11910,7 @@ namespace basist return false; } } - + if ((pHeader->m_slice_desc_file_ofs >= data_size) || ((data_size - pHeader->m_slice_desc_file_ofs) < (sizeof(basis_slice_desc) * pHeader->m_total_slices)) ) @@ -12028,20 +12026,20 @@ namespace basist image_info.m_image_index = image_index; image_info.m_total_levels = total_levels; - + image_info.m_alpha_flag = false; // For ETC1S, if anything has alpha all images have alpha. For UASTC, we only report alpha when the image actually has alpha. if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) - image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; + image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; else image_info.m_alpha_flag = (slice_desc.m_flags & cSliceDescFlagsHasAlpha) != 0; image_info.m_iframe_flag = (slice_desc.m_flags & cSliceDescFlagsFrameIsIFrame) != 0; - + const uint32_t block_width = basis_tex_format_get_block_width((basis_tex_format)((uint32_t)pHeader->m_tex_format)); const uint32_t block_height = basis_tex_format_get_block_height((basis_tex_format)((uint32_t)pHeader->m_tex_format)); - + image_info.m_width = slice_desc.m_num_blocks_x * block_width; image_info.m_height = slice_desc.m_num_blocks_y * block_height; image_info.m_orig_width = slice_desc.m_orig_width; @@ -12161,13 +12159,13 @@ namespace basist image_info.m_image_index = image_index; image_info.m_level_index = level_index; - + // For ETC1S, if anything has alpha all images have alpha. For UASTC, we only report alpha when the image actually has alpha. if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) image_info.m_alpha_flag = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; else image_info.m_alpha_flag = (slice_desc.m_flags & cSliceDescFlagsHasAlpha) != 0; - + const uint32_t block_width = basis_tex_format_get_block_width((basis_tex_format)((uint32_t)pHeader->m_tex_format)); const uint32_t block_height = basis_tex_format_get_block_height((basis_tex_format)((uint32_t)pHeader->m_tex_format)); @@ -12230,8 +12228,9 @@ namespace basist file_info.m_tex_format = static_cast(static_cast(pHeader->m_tex_format)); file_info.m_etc1s = (pHeader->m_tex_format == (int)basis_tex_format::cETC1S); - + file_info.m_y_flipped = (pHeader->m_flags & cBASISHeaderFlagYFlipped) != 0; + file_info.m_srgb = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; file_info.m_has_alpha_slices = (pHeader->m_flags & cBASISHeaderFlagHasAlphaSlices) != 0; const uint32_t total_slices = pHeader->m_total_slices; @@ -12302,7 +12301,7 @@ namespace basist return true; } - + bool basisu_transcoder::start_transcoding(const void* pData, uint32_t data_size) { if (!validate_header_quick(pData, data_size)) @@ -12410,7 +12409,7 @@ namespace basist m_lowlevel_etc1s_decoder.clear(); } } - + m_ready_to_transcode = true; return true; @@ -12421,7 +12420,7 @@ namespace basist m_lowlevel_etc1s_decoder.clear(); m_ready_to_transcode = false; - + return true; } @@ -12459,6 +12458,8 @@ namespace basist const basis_slice_desc& slice_desc = reinterpret_cast(pDataU8 + pHeader->m_slice_desc_file_ofs)[slice_index]; + const uint32_t dst_block_width = get_block_width(fmt), dst_block_height = get_block_height(fmt); + if (basis_block_format_is_uncompressed(fmt)) { // Assume the output buffer is orig_width by orig_height @@ -12487,33 +12488,23 @@ namespace basist return false; } } - else if (fmt == block_format::cASTC_HDR_6x6) - { - const uint32_t num_blocks_6x6_x = (slice_desc.m_orig_width + 5) / 6; - const uint32_t num_blocks_6x6_y = (slice_desc.m_orig_height + 5) / 6; - const uint32_t total_blocks_6x6 = num_blocks_6x6_x * num_blocks_6x6_y; - - if (output_blocks_buf_size_in_blocks_or_pixels < total_blocks_6x6) - { - BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: output_blocks_buf_size_in_blocks_or_pixels < total_blocks_6x6\n"); - return false; - } - } else { - // must be a 4x4 pixel block format - const uint32_t num_blocks_4x4_x = (slice_desc.m_orig_width + 3) / 4; - const uint32_t num_blocks_4x4_y = (slice_desc.m_orig_height + 3) / 4; - const uint32_t total_4x4_blocks = num_blocks_4x4_x * num_blocks_4x4_y; + const uint32_t dst_num_blocks_x = (slice_desc.m_orig_width + dst_block_width - 1) / dst_block_width; + const uint32_t dst_num_blocks_y = (slice_desc.m_orig_height + dst_block_height - 1) / dst_block_height; + const uint32_t dst_total_blocks = dst_num_blocks_x * dst_num_blocks_y; - if (output_blocks_buf_size_in_blocks_or_pixels < total_4x4_blocks) + if (output_blocks_buf_size_in_blocks_or_pixels < dst_total_blocks) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: output_blocks_buf_size_in_blocks_or_pixels < total_blocks\n"); return false; } } + + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); - if ((pHeader->m_tex_format == (uint32_t)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (uint32_t)basis_tex_format::cUASTC4x4)) + if ((pHeader->m_tex_format == (uint32_t)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (uint32_t)basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr) { if ((fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA)) { @@ -12538,16 +12529,18 @@ namespace basist BASISU_DEVEL_ERROR("basisu_transcoder::transcode_slice: invalid slice_desc.m_file_size, or passed in buffer too small\n"); return false; } - + if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6) { + // ASTC HDR 6x6 return m_lowlevel_astc_6x6_hdr_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { + // UASTC HDR 6x6 return m_lowlevel_astc_6x6_hdr_intermediate_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, @@ -12555,20 +12548,33 @@ namespace basist } else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 return m_lowlevel_uastc_4x4_hdr_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) { - return m_lowlevel_uastc_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, + // UASTC LDR 4x4 + return m_lowlevel_uastc_ldr_4x4_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, + pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, + fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, + output_rows_in_pixels, channel0, channel1, decode_flags); + } + else if ((is_xuastc_ldr) || (is_astc_ldr)) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + const bool use_astc_srgb_decode_profile = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; + + return m_lowlevel_xuastc_ldr_decoder.transcode_slice((basis_tex_format)(uint32_t)pHeader->m_tex_format, use_astc_srgb_decode_profile, pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); } else { + // must be ETC1S return m_lowlevel_etc1s_decoder.transcode_slice(pOutput_blocks, slice_desc.m_num_blocks_x, slice_desc.m_num_blocks_y, pDataU8 + slice_desc.m_file_ofs, slice_desc.m_file_size, fmt, output_block_or_pixel_stride_in_bytes, (decode_flags & cDecodeFlagsBC1ForbidThreeColorBlocks) == 0, *pHeader, slice_desc, output_row_pitch_in_blocks_or_pixels, pState, @@ -12647,7 +12653,7 @@ namespace basist if (!output_row_pitch_in_blocks_or_pixels) output_row_pitch_in_blocks_or_pixels = num_blocks_x; - + if ((fmt == block_format::cETC2_EAC_A8) || (fmt == block_format::cETC2_EAC_R11)) { #if BASISD_SUPPORT_ETC2_EAC_A8 @@ -12733,7 +12739,7 @@ namespace basist if (slice_index < 0) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: failed finding slice index\n"); - // Unable to find the requested image/level + // Unable to find the requested image/level return false; } @@ -12742,7 +12748,7 @@ namespace basist // Switch to PVRTC1 RGB if the input doesn't have alpha. fmt = transcoder_texture_format::cTFPVRTC1_4_RGB; } - + if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S) { if (pSlice_descs[slice_index].m_flags & cSliceDescFlagsHasAlpha) @@ -12779,10 +12785,13 @@ namespace basist } } } - + bool status = false; - if ((pHeader->m_tex_format == (int)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4)) + const bool is_xuastc_ldr = basis_tex_format_is_xuastc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + const bool is_astc_ldr = basis_tex_format_is_astc_ldr((basis_tex_format)(uint32_t)pHeader->m_tex_format); + + if ((pHeader->m_tex_format == (int)basis_tex_format::cETC1S) || (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) || is_xuastc_ldr || is_astc_ldr) { // Only do this on 4x4 LDR formats that supports transcoding to PVRTC1. const uint32_t total_slice_blocks = pSlice_descs[slice_index].m_num_blocks_x * pSlice_descs[slice_index].m_num_blocks_y; @@ -12790,28 +12799,28 @@ namespace basist if (((fmt == transcoder_texture_format::cTFPVRTC1_4_RGB) || (fmt == transcoder_texture_format::cTFPVRTC1_4_RGBA)) && (output_blocks_buf_size_in_blocks_or_pixels > total_slice_blocks)) { // The transcoder doesn't write beyond total_slice_blocks, so we need to clear the rest ourselves. - // For GL usage, PVRTC1 4bpp image size is (max(width, 8)* max(height, 8) * 4 + 7) / 8. + // For GL usage, PVRTC1 4bpp image size is (max(width, 8)* max(height, 8) * 4 + 7) / 8. // However, for KTX and internally in Basis this formula isn't used, it's just ((width+3)/4) * ((height+3)/4) * bytes_per_block_or_pixel. This is all the transcoder actually writes to memory. memset(static_cast(pOutput_blocks) + total_slice_blocks * bytes_per_block_or_pixel, 0, (output_blocks_buf_size_in_blocks_or_pixels - total_slice_blocks) * bytes_per_block_or_pixel); } } - + if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6) { + // ASTC HDR 6x6 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_astc_6x6_hdr_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { + // UASTC HDR 6x6 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_astc_6x6_hdr_intermediate_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, @@ -12820,28 +12829,50 @@ namespace basist } else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. status = m_lowlevel_uastc_4x4_hdr_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) + else if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC_LDR_4x4) { + // UASTC LDR 4x4 const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; - // Use the container independent image transcode method. - status = m_lowlevel_uastc_decoder.transcode_image(fmt, + status = m_lowlevel_uastc_ldr_4x4_decoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } - else + else if (is_xuastc_ldr || is_astc_ldr) { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; + + const bool use_astc_srgb_decode_profile = (pHeader->m_flags & cBASISHeaderFlagSRGB) != 0; + + status = m_lowlevel_xuastc_ldr_decoder.transcode_image((basis_tex_format)(uint32_t)pHeader->m_tex_format, use_astc_srgb_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + (const uint8_t*)pData, data_size, pSlice_desc->m_num_blocks_x, pSlice_desc->m_num_blocks_y, pSlice_desc->m_orig_width, pSlice_desc->m_orig_height, pSlice_desc->m_level_index, + pSlice_desc->m_file_ofs, pSlice_desc->m_file_size, + decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); + } + else + { + // ETC1S + + // sanity check + if (pHeader->m_tex_format != (uint32_t)basis_tex_format::cETC1S) + { + BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: unsupported texture format\n"); + return false; + } + // ETC1S const basis_slice_desc* pSlice_desc = &pSlice_descs[slice_index]; const basis_slice_desc* pAlpha_slice_desc = basis_file_has_alpha_slices ? &pSlice_descs[slice_index + 1] : nullptr; @@ -12866,14 +12897,14 @@ namespace basist decode_flags, basis_file_has_alpha_slices, pHeader->m_tex_type == cBASISTexTypeVideoFrames, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels); } // if (pHeader->m_tex_format == (int)basis_tex_format::cUASTC4x4) - + if (!status) { BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning false\n"); } else { - //BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning true\n"); + //BASISU_DEVEL_ERROR("basisu_transcoder::transcode_image_level: Returning true\n"); } return status; @@ -12899,7 +12930,22 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: case transcoder_texture_format::cTFBC3_RGBA: case transcoder_texture_format::cTFBC5_RG: - case transcoder_texture_format::cTFASTC_4x4_RGBA: + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: case transcoder_texture_format::cTFATC_RGBA: @@ -12939,7 +12985,7 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return "ETC2_RGBA"; case transcoder_texture_format::cTFBC3_RGBA: return "BC3_RGBA"; case transcoder_texture_format::cTFBC5_RG: return "BC5_RG"; - case transcoder_texture_format::cTFASTC_4x4_RGBA: return "ASTC_RGBA"; + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: return "ASTC_HDR_4X4_RGBA"; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return "ASTC_HDR_6X6_RGBA"; case transcoder_texture_format::cTFATC_RGB: return "ATC_RGB"; @@ -12957,6 +13003,22 @@ namespace basist case transcoder_texture_format::cTFETC2_EAC_R11: return "ETC2_EAC_R11"; case transcoder_texture_format::cTFETC2_EAC_RG11: return "ETC2_EAC_RG11"; case transcoder_texture_format::cTFBC6H: return "BC6H"; + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return "ASTC_LDR_4X4_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return "ASTC_LDR_5X4_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return "ASTC_LDR_5X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return "ASTC_LDR_6X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return "ASTC_LDR_6X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return "ASTC_LDR_8X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return "ASTC_LDR_8X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return "ASTC_LDR_10X5_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return "ASTC_LDR_10X6_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return "ASTC_LDR_8X8_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return "ASTC_LDR_10X8_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return "ASTC_LDR_10X10_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return "ASTC_LDR_12X10_RGBA"; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return "ASTC_LDR_12X12_RGBA"; + default: assert(0); BASISU_DEVEL_ERROR("basis_get_basisu_texture_format: Invalid fmt\n"); @@ -12965,6 +13027,51 @@ namespace basist return ""; } + const char* basis_get_tex_format_name(basis_tex_format fmt) + { + switch (fmt) + { + case basis_tex_format::cETC1S: return "ETC1S"; break; + case basis_tex_format::cUASTC_LDR_4x4: return "UASTC LDR 4x4"; break; + case basis_tex_format::cUASTC_HDR_4x4: return "UASTC_HDR_4x4"; break; + case basis_tex_format::cASTC_HDR_6x6: return "ASTC_HDR_6x6"; break; + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return "UASTC_HDR_6x6"; break; + case basis_tex_format::cXUASTC_LDR_4x4: return "XUASTC LDR 4x4"; break; + case basis_tex_format::cXUASTC_LDR_5x4: return "XUASTC LDR 5x4"; break; + case basis_tex_format::cXUASTC_LDR_5x5: return "XUASTC LDR 5x5"; break; + case basis_tex_format::cXUASTC_LDR_6x5: return "XUASTC LDR 6x5"; break; + case basis_tex_format::cXUASTC_LDR_6x6: return "XUASTC LDR 6x6"; break; + case basis_tex_format::cXUASTC_LDR_8x5: return "XUASTC LDR 8x5"; break; + case basis_tex_format::cXUASTC_LDR_8x6: return "XUASTC LDR 8x6"; break; + case basis_tex_format::cXUASTC_LDR_10x5: return "XUASTC LDR 10x5"; break; + case basis_tex_format::cXUASTC_LDR_10x6: return "XUASTC LDR 10x6"; break; + case basis_tex_format::cXUASTC_LDR_8x8: return "XUASTC LDR 8x8"; break; + case basis_tex_format::cXUASTC_LDR_10x8: return "XUASTC LDR 10x8"; break; + case basis_tex_format::cXUASTC_LDR_10x10: return "XUASTC LDR 10x10"; break; + case basis_tex_format::cXUASTC_LDR_12x10: return "XUASTC LDR 12x10"; break; + case basis_tex_format::cXUASTC_LDR_12x12: return "XUASTC LDR 12x12"; break; + case basis_tex_format::cASTC_LDR_4x4: return "ASTC LDR 4x4"; break; + case basis_tex_format::cASTC_LDR_5x4: return "ASTC LDR 5x4"; break; + case basis_tex_format::cASTC_LDR_5x5: return "ASTC LDR 5x5"; break; + case basis_tex_format::cASTC_LDR_6x5: return "ASTC LDR 6x5"; break; + case basis_tex_format::cASTC_LDR_6x6: return "ASTC LDR 6x6"; break; + case basis_tex_format::cASTC_LDR_8x5: return "ASTC LDR 8x5"; break; + case basis_tex_format::cASTC_LDR_8x6: return "ASTC LDR 8x6"; break; + case basis_tex_format::cASTC_LDR_10x5: return "ASTC LDR 10x5"; break; + case basis_tex_format::cASTC_LDR_10x6: return "ASTC LDR 10x6"; break; + case basis_tex_format::cASTC_LDR_8x8: return "ASTC LDR 8x8"; break; + case basis_tex_format::cASTC_LDR_10x8: return "ASTC LDR 10x8"; break; + case basis_tex_format::cASTC_LDR_10x10: return "ASTC LDR 10x10"; break; + case basis_tex_format::cASTC_LDR_12x10: return "ASTC LDR 12x10"; break; + case basis_tex_format::cASTC_LDR_12x12: return "ASTC LDR 12x12"; break; + default: + assert(0); + BASISU_DEVEL_ERROR("basis_get_tex_format_name: Invalid parameter\n"); + break; + } + return ""; + } + const char* basis_get_block_format_name(block_format fmt) { switch (fmt) @@ -12976,7 +13083,22 @@ namespace basist case block_format::cBC7: return "BC7"; case block_format::cETC2_RGBA: return "ETC2_RGBA"; case block_format::cBC3: return "BC3"; - case block_format::cASTC_4x4: return "ASTC_4x4"; + + case block_format::cASTC_LDR_4x4: return "ASTC_LDR_4x4"; + case block_format::cASTC_LDR_5x4: return "ASTC_LDR_5x4"; + case block_format::cASTC_LDR_5x5: return "ASTC_LDR_5x5"; + case block_format::cASTC_LDR_6x5: return "ASTC_LDR_6x5"; + case block_format::cASTC_LDR_6x6: return "ASTC_LDR_6x6"; + case block_format::cASTC_LDR_8x5: return "ASTC_LDR_8x5"; + case block_format::cASTC_LDR_8x6: return "ASTC_LDR_8x6"; + case block_format::cASTC_LDR_10x5: return "ASTC_LDR_10x5"; + case block_format::cASTC_LDR_10x6: return "ASTC_LDR_10x6"; + case block_format::cASTC_LDR_8x8: return "ASTC_LDR_8x8"; + case block_format::cASTC_LDR_10x8: return "ASTC_LDR_10x8"; + case block_format::cASTC_LDR_10x10: return "ASTC_LDR_10x10"; + case block_format::cASTC_LDR_12x10: return "ASTC_LDR_12x10"; + case block_format::cASTC_LDR_12x12: return "ASTC_LDR_12x12"; + case block_format::cATC_RGB: return "ATC_RGB"; case block_format::cRGBA32: return "RGBA32"; case block_format::cRGB565: return "RGB565"; @@ -13027,7 +13149,22 @@ namespace basist { case transcoder_texture_format::cTFETC2_RGBA: case transcoder_texture_format::cTFBC3_RGBA: - case transcoder_texture_format::cTFASTC_4x4_RGBA: + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: // technically this ASTC HDR format supports alpha, but we currently don't exploit that in our encoders case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: // technically this ASTC HDR format supports alpha, but we currently don't exploit that in our encoders case transcoder_texture_format::cTFBC7_RGBA: @@ -13062,6 +13199,33 @@ namespace basist return false; } + bool basis_is_transcoder_texture_format_astc(transcoder_texture_format fmt) + { + switch (fmt) + { + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: + return true; + default: + break; + } + return false; + } + basisu::texture_format basis_get_basisu_texture_format(transcoder_texture_format fmt) { switch (fmt) @@ -13076,7 +13240,22 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return basisu::texture_format::cETC2_RGBA; case transcoder_texture_format::cTFBC3_RGBA: return basisu::texture_format::cBC3; case transcoder_texture_format::cTFBC5_RG: return basisu::texture_format::cBC5; - case transcoder_texture_format::cTFASTC_4x4_RGBA: return basisu::texture_format::cASTC_LDR_4x4; + + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return basisu::texture_format::cASTC_LDR_4x4; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return basisu::texture_format::cASTC_LDR_5x4; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return basisu::texture_format::cASTC_LDR_5x5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return basisu::texture_format::cASTC_LDR_6x5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return basisu::texture_format::cASTC_LDR_6x6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return basisu::texture_format::cASTC_LDR_8x5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return basisu::texture_format::cASTC_LDR_8x6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return basisu::texture_format::cASTC_LDR_10x5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return basisu::texture_format::cASTC_LDR_10x6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return basisu::texture_format::cASTC_LDR_8x8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return basisu::texture_format::cASTC_LDR_10x8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return basisu::texture_format::cASTC_LDR_10x10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return basisu::texture_format::cASTC_LDR_12x10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return basisu::texture_format::cASTC_LDR_12x12; + case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: return basisu::texture_format::cASTC_HDR_4x4; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return basisu::texture_format::cASTC_HDR_6x6; case transcoder_texture_format::cTFBC6H: return basisu::texture_format::cBC6HUnsigned; @@ -13142,14 +13321,14 @@ namespace basist } return false; } - + uint32_t basis_get_uncompressed_bytes_per_pixel(transcoder_texture_format fmt) { switch (fmt) { case transcoder_texture_format::cTFRGBA32: case transcoder_texture_format::cTFRGB_9E5: - return sizeof(uint32_t); + return sizeof(uint32_t); case transcoder_texture_format::cTFRGB565: case transcoder_texture_format::cTFBGR565: case transcoder_texture_format::cTFRGBA4444: @@ -13163,27 +13342,57 @@ namespace basist } return 0; } - - uint32_t basis_get_block_width(transcoder_texture_format tex_type) + + uint32_t basis_get_block_width(transcoder_texture_format fmt) { - switch (tex_type) + switch (fmt) { case transcoder_texture_format::cTFFXT1_RGB: return 8; case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return 6; + + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return 12; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return 12; + default: break; } return 4; } - uint32_t basis_get_block_height(transcoder_texture_format tex_type) + uint32_t basis_get_block_height(transcoder_texture_format fmt) { - switch (tex_type) + switch (fmt) { case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: return 6; + + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return 5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return 6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return 8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return 10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return 12; + + default: break; } @@ -13195,8 +13404,36 @@ namespace basist switch (fmt) { case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return 6; + case basis_tex_format::cXUASTC_LDR_4x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x4: return 5; + case basis_tex_format::cXUASTC_LDR_5x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x5: return 6; + case basis_tex_format::cXUASTC_LDR_6x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x5: return 8; + case basis_tex_format::cXUASTC_LDR_8x6: return 8; + case basis_tex_format::cXUASTC_LDR_10x5: return 10; + case basis_tex_format::cXUASTC_LDR_10x6: return 10; + case basis_tex_format::cXUASTC_LDR_8x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x8: return 10; + case basis_tex_format::cXUASTC_LDR_10x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x10: return 12; + case basis_tex_format::cXUASTC_LDR_12x12: return 12; + case basis_tex_format::cASTC_LDR_4x4: return 4; + case basis_tex_format::cASTC_LDR_5x4: return 5; + case basis_tex_format::cASTC_LDR_5x5: return 5; + case basis_tex_format::cASTC_LDR_6x5: return 6; + case basis_tex_format::cASTC_LDR_6x6: return 6; + case basis_tex_format::cASTC_LDR_8x5: return 8; + case basis_tex_format::cASTC_LDR_8x6: return 8; + case basis_tex_format::cASTC_LDR_10x5: return 10; + case basis_tex_format::cASTC_LDR_10x6: return 10; + case basis_tex_format::cASTC_LDR_8x8: return 8; + case basis_tex_format::cASTC_LDR_10x8: return 10; + case basis_tex_format::cASTC_LDR_10x10: return 10; + case basis_tex_format::cASTC_LDR_12x10: return 12; + case basis_tex_format::cASTC_LDR_12x12: return 12; default: break; } @@ -13208,8 +13445,36 @@ namespace basist switch (fmt) { case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return 6; + case basis_tex_format::cXUASTC_LDR_4x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x4: return 4; + case basis_tex_format::cXUASTC_LDR_5x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x5: return 5; + case basis_tex_format::cXUASTC_LDR_6x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x5: return 5; + case basis_tex_format::cXUASTC_LDR_8x6: return 6; + case basis_tex_format::cXUASTC_LDR_10x5: return 5; + case basis_tex_format::cXUASTC_LDR_10x6: return 6; + case basis_tex_format::cXUASTC_LDR_8x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x8: return 8; + case basis_tex_format::cXUASTC_LDR_10x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x10: return 10; + case basis_tex_format::cXUASTC_LDR_12x12: return 12; + case basis_tex_format::cASTC_LDR_4x4: return 4; + case basis_tex_format::cASTC_LDR_5x4: return 4; + case basis_tex_format::cASTC_LDR_5x5: return 5; + case basis_tex_format::cASTC_LDR_6x5: return 5; + case basis_tex_format::cASTC_LDR_6x6: return 6; + case basis_tex_format::cASTC_LDR_8x5: return 5; + case basis_tex_format::cASTC_LDR_8x6: return 6; + case basis_tex_format::cASTC_LDR_10x5: return 5; + case basis_tex_format::cASTC_LDR_10x6: return 6; + case basis_tex_format::cASTC_LDR_8x8: return 8; + case basis_tex_format::cASTC_LDR_10x8: return 8; + case basis_tex_format::cASTC_LDR_10x10: return 10; + case basis_tex_format::cASTC_LDR_12x10: return 10; + case basis_tex_format::cASTC_LDR_12x12: return 12; default: break; } @@ -13222,7 +13487,7 @@ namespace basist { case basis_tex_format::cUASTC_HDR_4x4: case basis_tex_format::cASTC_HDR_6x6: - case basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: return true; default: break; @@ -13230,9 +13495,136 @@ namespace basist return false; } + // Given a basis_tex_format (mode or codec), return the corresponding ASTC texture_format with the proper block size from 4x4-12x12. + basisu::texture_format basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt) + { + switch (fmt) + { + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + return basisu::texture_format::cASTC_LDR_4x4; + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + return basisu::texture_format::cASTC_LDR_5x4; + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + return basisu::texture_format::cASTC_LDR_5x5; + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + return basisu::texture_format::cASTC_LDR_6x5; + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + return basisu::texture_format::cASTC_LDR_6x6; + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + return basisu::texture_format::cASTC_LDR_8x5; + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + return basisu::texture_format::cASTC_LDR_8x6; + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + return basisu::texture_format::cASTC_LDR_10x5; + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + return basisu::texture_format::cASTC_LDR_10x6; + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + return basisu::texture_format::cASTC_LDR_8x8; + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + return basisu::texture_format::cASTC_LDR_10x8; + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + return basisu::texture_format::cASTC_LDR_10x10; + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + return basisu::texture_format::cASTC_LDR_12x10; + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + return basisu::texture_format::cASTC_LDR_12x12; + default: + assert(0); + return basisu::texture_format::cInvalidTextureFormat; + } + } + + // Given any basis_tex_format (mode or codec), return the corresponding transcoder_texture_format with the proper ASTC block size from 4x4-12x12. + transcoder_texture_format basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt) + { + switch (fmt) + { + // XUASTC 4x4-12x12 and ASTC 4x4-12x12 + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + return transcoder_texture_format::cTFASTC_LDR_5x4_RGBA; + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + return transcoder_texture_format::cTFASTC_LDR_5x5_RGBA; + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + return transcoder_texture_format::cTFASTC_LDR_6x5_RGBA; + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + return transcoder_texture_format::cTFASTC_LDR_6x6_RGBA; + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + return transcoder_texture_format::cTFASTC_LDR_8x5_RGBA; + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + return transcoder_texture_format::cTFASTC_LDR_8x6_RGBA; + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + return transcoder_texture_format::cTFASTC_LDR_10x5_RGBA; + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + return transcoder_texture_format::cTFASTC_LDR_10x6_RGBA; + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + return transcoder_texture_format::cTFASTC_LDR_8x8_RGBA; + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + return transcoder_texture_format::cTFASTC_LDR_10x8_RGBA; + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + return transcoder_texture_format::cTFASTC_LDR_10x10_RGBA; + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + return transcoder_texture_format::cTFASTC_LDR_12x10_RGBA; + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + return transcoder_texture_format::cTFASTC_LDR_12x12_RGBA; + + // ETC1S/UASTC LDR 4x4 + case basis_tex_format::cETC1S: + case basis_tex_format::cUASTC_LDR_4x4: + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + + // HDR formats + case basis_tex_format::cUASTC_HDR_4x4: + return transcoder_texture_format::cTFASTC_HDR_4x4_RGBA; + + case basis_tex_format::cASTC_HDR_6x6: + case basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE: + return transcoder_texture_format::cTFASTC_HDR_6x6_RGBA; + + default: + assert(0); + return transcoder_texture_format::cTFASTC_LDR_4x4_RGBA; + } + } + + transcoder_texture_format basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format fmt) + { + return basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(fmt); + } + + // For a given basis_tex_format (mode or codec), is the specified transcoder_texture_format supported? bool basis_is_format_supported(transcoder_texture_format tex_type, basis_tex_format fmt) { - if ((fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE)) + if ((fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE)) { // RDO UASTC HDR 6x6, or our custom intermediate format #if BASISD_SUPPORT_UASTC_HDR @@ -13266,37 +13658,182 @@ namespace basist } #endif } - else if (fmt == basis_tex_format::cUASTC4x4) + else if (fmt == basis_tex_format::cUASTC_LDR_4x4) { // UASTC LDR 4x4 #if BASISD_SUPPORT_UASTC + // IMPORTANT : This is defined as the formats which DON'T support UASTC LDR 4x4 transcoding. switch (tex_type) { - // These niche formats aren't currently supported for UASTC - everything else is. + // These niche formats aren't currently supported for UASTC LDR 4x4 - everything else is. case transcoder_texture_format::cTFPVRTC2_4_RGB: case transcoder_texture_format::cTFPVRTC2_4_RGBA: case transcoder_texture_format::cTFATC_RGB: case transcoder_texture_format::cTFATC_RGBA: case transcoder_texture_format::cTFFXT1_RGB: - // UASTC LDR doesn't support transcoding to HDR formats + // UASTC LDR 4x4 doesn't support transcoding to HDR formats case transcoder_texture_format::cTFASTC_HDR_4x4_RGBA: case transcoder_texture_format::cTFASTC_HDR_6x6_RGBA: case transcoder_texture_format::cTFBC6H: case transcoder_texture_format::cTFRGBA_HALF: case transcoder_texture_format::cTFRGB_HALF: case transcoder_texture_format::cTFRGB_9E5: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return false; default: return true; } #endif } + else if ( (basis_tex_format_is_xuastc_ldr(fmt)) || (basis_tex_format_is_astc_ldr(fmt)) ) + { + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + switch (tex_type) + { + case transcoder_texture_format::cTFBC1_RGB: + case transcoder_texture_format::cTFBC3_RGBA: + case transcoder_texture_format::cTFBC4_R: + case transcoder_texture_format::cTFBC5_RG: + case transcoder_texture_format::cTFBC7_RGBA: + case transcoder_texture_format::cTFETC1_RGB: + case transcoder_texture_format::cTFETC2_RGBA: + case transcoder_texture_format::cTFETC2_EAC_R11: + case transcoder_texture_format::cTFETC2_EAC_RG11: + case transcoder_texture_format::cTFPVRTC1_4_RGB: + case transcoder_texture_format::cTFPVRTC1_4_RGBA: + // Uncompressed formats + case transcoder_texture_format::cTFRGBA32: + case transcoder_texture_format::cTFRGB565: + case transcoder_texture_format::cTFBGR565: + case transcoder_texture_format::cTFRGBA4444: + return true; + default: + break; + } + + // Ensure they're using the block size for ASTC LDR that matches the XUASTC format's block size. + switch (fmt) + { + case basis_tex_format::cXUASTC_LDR_4x4: + case basis_tex_format::cASTC_LDR_4x4: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_4x4_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_5x4: + case basis_tex_format::cASTC_LDR_5x4: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_5x4_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_5x5: + case basis_tex_format::cASTC_LDR_5x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_5x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_6x5: + case basis_tex_format::cASTC_LDR_6x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_6x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_6x6: + case basis_tex_format::cASTC_LDR_6x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_6x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x5: + case basis_tex_format::cASTC_LDR_8x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x6: + case basis_tex_format::cASTC_LDR_8x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x5: + case basis_tex_format::cASTC_LDR_10x5: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x5_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x6: + case basis_tex_format::cASTC_LDR_10x6: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x6_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_8x8: + case basis_tex_format::cASTC_LDR_8x8: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_8x8_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x8: + case basis_tex_format::cASTC_LDR_10x8: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x8_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_10x10: + case basis_tex_format::cASTC_LDR_10x10: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_10x10_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_12x10: + case basis_tex_format::cASTC_LDR_12x10: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_12x10_RGBA) + return true; + break; + } + case basis_tex_format::cXUASTC_LDR_12x12: + case basis_tex_format::cASTC_LDR_12x12: + { + if (tex_type == transcoder_texture_format::cTFASTC_LDR_12x12_RGBA) + return true; + break; + } + default: + break; + } + } else { // ETC1S switch (tex_type) { - // ETC1 and uncompressed are always supported. + // ETC1 and uncompressed are always supported. case transcoder_texture_format::cTFETC1_RGB: case transcoder_texture_format::cTFRGBA32: case transcoder_texture_format::cTFRGB565: @@ -13330,8 +13867,8 @@ namespace basist case transcoder_texture_format::cTFETC2_RGBA: return true; #endif -#if BASISD_SUPPORT_ASTC - case transcoder_texture_format::cTFASTC_4x4_RGBA: +#if BASISD_SUPPORT_ASTC + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return true; #endif #if BASISD_SUPPORT_ATC @@ -13361,9 +13898,9 @@ namespace basist return false; } - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ // UASTC LDR 4x4 - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_UASTC const astc_bc7_common_partition2_desc g_astc_bc7_common_partitions2[TOTAL_ASTC_BC7_COMMON_PARTITIONS2] = @@ -14088,7 +14625,7 @@ namespace basist if (group_size) { - // Range has trits or quints - pack each group of 5 or 3 values + // Range has trits or quints - pack each group of 5 or 3 values const int total_groups = (group_size == 5) ? ((num_vals + 4) / 5) : ((num_vals + 2) / 3); for (int group_index = 0; group_index < total_groups; group_index++) @@ -14380,7 +14917,7 @@ namespace basist bool unpack_uastc(const uastc_block& blk, unpacked_uastc_block& unpacked, bool blue_contract_check, bool read_hints) { //memset(&unpacked, 0, sizeof(unpacked)); - + #if 0 uint8_t table[128]; memset(table, 0xFF, sizeof(table)); @@ -14436,7 +14973,7 @@ namespace basist return true; } - + if (read_hints) { if (g_uastc_mode_has_bc1_hint0[mode]) @@ -14469,7 +15006,7 @@ namespace basist } else bit_ofs += g_uastc_mode_total_hint_bits[mode]; - + uint32_t subsets = 1; switch (mode) { @@ -14682,7 +15219,7 @@ namespace basist { // All other modes have <= 64 weight bits. uint64_t bits; - + // Read the weight bits if ((BASISD_IS_BIG_ENDIAN) || (!BASISD_USE_UNALIGNED_WORD_READS)) bits = read_bits64(blk.m_bytes, bit_ofs, basisu::minimum(64, 128 - (int)bit_ofs)); @@ -14690,31 +15227,31 @@ namespace basist { bits = blk.m_dwords[2]; bits |= (((uint64_t)blk.m_dwords[3]) << 32U); - + if (bit_ofs >= 64U) bits >>= (bit_ofs - 64U); else { assert(bit_ofs >= 56U); - + uint32_t bits_needed = 64U - bit_ofs; bits <<= bits_needed; bits |= (blk.m_bytes[7] >> (8U - bits_needed)); } } - + bit_ofs = 0; const uint32_t mask = (1U << weight_bits) - 1U; const uint32_t anchor_mask = (1U << (weight_bits - 1U)) - 1U; - + if (total_planes == 2) { // Dual plane modes always have a single subset, and the first 2 weights are anchors. unpacked.m_astc.m_weights[0] = (uint8_t)((uint32_t)(bits >> bit_ofs) & anchor_mask); bit_ofs += (weight_bits - 1); - + unpacked.m_astc.m_weights[1] = (uint8_t)((uint32_t)(bits >> bit_ofs) & anchor_mask); bit_ofs += (weight_bits - 1); @@ -14732,7 +15269,7 @@ namespace basist if (weight_bits == 4) { assert(bit_ofs == 0); - + // Specialize the most common case: 4-bit weights. unpacked.m_astc.m_weights[0] = (uint8_t)((uint32_t)(bits) & 7); unpacked.m_astc.m_weights[1] = (uint8_t)((uint32_t)(bits >> 3) & 15); @@ -14991,7 +15528,7 @@ namespace basist return unpack_uastc(unpacked_blk, pPixels, srgb); } - // Determines the best shared pbits to use to encode xl/xh + // Determines the best shared pbit to use to encode xl/xh static void determine_shared_pbits( uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], color_quad_u8& bestMinColor, color_quad_u8& bestMaxColor, uint32_t best_pbits[2]) @@ -15195,6 +15732,8 @@ namespace basist } uint32_t best_pbits[2]; + basisu::clear_obj(best_pbits); + color_quad_u8 bestMinColor, bestMaxColor; determine_unique_pbits((total_comps == 2) ? 4 : total_comps, 7, xl, xh, bestMinColor, bestMaxColor, best_pbits); @@ -15278,7 +15817,7 @@ namespace basist } case 2: { - // 2. DualPlane: 0, WeightRange : 5 (8), Subsets : 2, EndpointRange : 8 (16) - BC7 MODE1 + // 2. DualPlane: 0, WeightRange : 5 (8), Subsets : 2, EndpointRange : 8 (16) - BC7 MODE1 dst_blk.m_mode = 1; dst_blk.m_partition = g_astc_bc7_common_partitions2[unpacked_src_blk.m_common_pattern].m_bc7; @@ -15505,6 +16044,7 @@ namespace basist case UASTC_MODE_INDEX_SOLID_COLOR: { // Void-Extent: Solid Color RGBA (BC7 MODE5 or MODE6) + // TODO: Why prefer mode 6 here? Mode 5 is lossless. const color32& solid_color = unpacked_src_blk.m_solid_color; uint32_t best_err0 = g_bc7_mode_6_optimal_endpoints[solid_color.r][0].m_error + g_bc7_mode_6_optimal_endpoints[solid_color.g][0].m_error + @@ -16217,7 +16757,7 @@ namespace basist bool flip = pack_etc1_y_estimate_flipped(&block_y[0][0], upper_avg, lower_avg, left_avg, right_avg); // non-flipped: | | - // vs. + // vs. // flipped: -- // -- @@ -16828,7 +17368,7 @@ namespace basist static const uint8_t s_uastc2_to_bc1[4] = { 0, 2, 3, 1 }; static const uint8_t s_uastc1_to_bc1[2] = { 0, 1 }; const uint8_t* s_uastc_to_bc1_weights[6] = { nullptr, s_uastc1_to_bc1, s_uastc2_to_bc1, s_uastc3_to_bc1, s_uastc4_to_bc1, s_uastc5_to_bc1 }; - + void encode_bc4(void* pDst, const uint8_t* pPixels, uint32_t stride) { uint32_t min0_v, max0_v, min1_v, max1_v,min2_v, max2_v, min3_v, max3_v; @@ -16916,7 +17456,7 @@ namespace basist a2 |= (s_tran2[(v2 >= t0) + (v2 >= t1) + (v2 >= t2) + (v2 >= t3) + (v2 >= t4) + (v2 >= t5) + (v2 >= t6)] << 12U); a3 |= (s_tran3[(v3 >= t0) + (v3 >= t1) + (v3 >= t2) + (v3 >= t3) + (v3 >= t4) + (v3 >= t5) + (v3 >= t6)] << 12U); } - + { const int v0 = pPixels[8 * stride] * 14 + bias; const int v1 = pPixels[9 * stride] * 14 + bias; @@ -16940,7 +17480,7 @@ namespace basist } const uint64_t f = a0 | a1 | a2 | a3; - + pDst_bytes[2] = (uint8_t)f; pDst_bytes[3] = (uint8_t)(f >> 8U); pDst_bytes[4] = (uint8_t)(f >> 16U); @@ -16963,7 +17503,7 @@ namespace basist int dots[4]; for (uint32_t i = 0; i < 4; i++) dots[i] = (int)block_r[i] * ar + (int)block_g[i] * ag + (int)block_b[i] * ab; - + int t0 = dots[0] + dots[1], t1 = dots[1] + dots[2], t2 = dots[2] + dots[3]; ar *= 2; ag *= 2; ab *= 2; @@ -16972,7 +17512,7 @@ namespace basist { const int d = pSrc_pixels[i].r * ar + pSrc_pixels[i].g * ag + pSrc_pixels[i].b * ab; static const uint8_t s_sels[4] = { 3, 2, 1, 0 }; - + // Rounding matters here! // d <= t0: <=, not <, to the later LS step "sees" a wider range of selectors. It matters for quality. sels[i] = s_sels[(d <= t0) + (d < t1) + (d < t2)]; @@ -17013,11 +17553,11 @@ namespace basist sels[i+3] = s_sels[(d3 <= t0) + (d3 < t1) + (d3 < t2)]; } } - + static bool compute_least_squares_endpoints_rgb(const color32* pColors, const uint8_t* pSelectors, vec3F* pXl, vec3F* pXh) { // Derived from bc7enc16's LS function. - // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf + // Least squares using normal equations: http://www.cs.cornell.edu/~bindel/class/cs3220-s12/notes/lec10.pdf // I did this in matrix form first, expanded out all the ops, then optimized it a bit. uint32_t uq00_r = 0, uq10_r = 0, ut_r = 0, uq00_g = 0, uq10_g = 0, ut_g = 0, uq00_b = 0, uq10_b = 0, ut_b = 0; @@ -17091,7 +17631,7 @@ namespace basist return true; } - void encode_bc1_solid_block(void* pDst, uint32_t fr, uint32_t fg, uint32_t fb) + void encode_bc1_solid_block(void* pDst, uint32_t fr, uint32_t fg, uint32_t fb) { dxt1_block* pDst_block = static_cast(pDst); @@ -17143,19 +17683,19 @@ namespace basist { const color32* pSrc_pixels = (const color32*)pPixels; dxt1_block* pDst_block = static_cast(pDst); - + int avg_r = -1, avg_g = 0, avg_b = 0; int lr = 0, lg = 0, lb = 0, hr = 0, hg = 0, hb = 0; uint8_t sels[16]; - + const bool use_sels = (flags & cEncodeBC1UseSelectors) != 0; if (use_sels) { // Caller is jamming in their own selectors for us to try. const uint32_t s = pDst_block->m_selectors[0] | (pDst_block->m_selectors[1] << 8) | (pDst_block->m_selectors[2] << 16) | (pDst_block->m_selectors[3] << 24); - + static const uint8_t s_sel_tran[4] = { 0, 3, 1, 2 }; - + for (uint32_t i = 0; i < 16; i++) sels[i] = s_sel_tran[(s >> (i * 2)) & 3]; } @@ -17167,13 +17707,13 @@ namespace basist for (j = 1; j < 16; j++) if ((pSrc_pixels[j].r != fr) || (pSrc_pixels[j].g != fg) || (pSrc_pixels[j].b != fb)) break; - + if (j == 16) { encode_bc1_solid_block(pDst, fr, fg, fb); return; } - + // Select 2 colors along the principle axis. (There must be a faster/simpler way.) int total_r = fr, total_g = fg, total_b = fb; int max_r = fr, max_g = fg, max_b = fb; @@ -17207,7 +17747,7 @@ namespace basist float cov[6]; for (uint32_t i = 0; i < 6; i++) cov[i] = static_cast(icov[i])* (1.0f / 255.0f); - + #if 0 // Seems silly to use full PCA to choose 2 colors. The diff in avg. PSNR between using PCA vs. not is small (~.025 difference). // TODO: Try 2 or 3 different normalized diagonal vectors, choose the one that results in the largest dot delta @@ -17239,7 +17779,7 @@ namespace basist saxis_b = (int)(xb * m); } #endif - + int low_dot = INT_MAX, high_dot = INT_MIN, low_c = 0, high_c = 0; for (uint32_t i = 0; i < 16; i++) { @@ -17263,7 +17803,7 @@ namespace basist hr = to_5(pSrc_pixels[high_c].r); hg = to_6(pSrc_pixels[high_c].g); hb = to_5(pSrc_pixels[high_c].b); - + bc1_find_sels(pSrc_pixels, lr, lg, lb, hr, hg, hb, sels); } // if (use_sels) @@ -17310,13 +17850,13 @@ namespace basist hg = basisu::clamp((int)((xh.c[1]) * (63.0f / 255.0f) + .5f), 0, 63); hb = basisu::clamp((int)((xh.c[2]) * (31.0f / 255.0f) + .5f), 0, 31); } - + bc1_find_sels(pSrc_pixels, lr, lg, lb, hr, hg, hb, sels); } uint32_t lc16 = dxt1_block::pack_unscaled_color(lr, lg, lb); uint32_t hc16 = dxt1_block::pack_unscaled_color(hr, hg, hb); - + // Always forbid 3 color blocks if (lc16 == hc16) { @@ -17368,7 +17908,7 @@ namespace basist pDst_block->m_selectors[3] = (uint8_t)(packed_sels >> 24) ^ invert_mask; } } - + void encode_bc1_alt(void* pDst, const uint8_t* pPixels, uint32_t flags) { const color32* pSrc_pixels = (const color32*)pPixels; @@ -17417,8 +17957,8 @@ namespace basist min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); total_r += r; total_g += g; total_b += b; } - - if (grayscale_flag) + + if (grayscale_flag) { // Grayscale blocks are a common enough case to specialize. if ((max_r - min_r) < 2) @@ -17735,7 +18275,7 @@ namespace basist // Always forbid 3 color blocks uint16_t lc16 = (uint16_t)b.get_low_color(); uint16_t hc16 = (uint16_t)b.get_high_color(); - + uint8_t mask = 0; // Make l > h @@ -17965,7 +18505,7 @@ namespace basist blk.m_base = static_cast(a); blk.m_table = 13; blk.m_multiplier = 0; - + memcpy(blk.m_selectors, g_etc2_eac_a8_sel4, sizeof(g_etc2_eac_a8_sel4)); return; @@ -18655,7 +19195,7 @@ namespace basist if (!unpack_uastc(pSrc_blocks[x + y * num_blocks_x], block_pixels, false)) return false; - // Get block's RGB bounding box + // Get block's RGB bounding box color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); if (from_alpha) @@ -18714,7 +19254,7 @@ namespace basist if (!unpack_uastc(pSrc_blocks[x + y * num_blocks_x], block_pixels, false)) return false; - // Get block's RGBA bounding box + // Get block's RGBA bounding box color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); for (uint32_t i = 0; i < 16; i++) @@ -18830,9 +19370,9 @@ namespace basist #endif // #if BASISD_SUPPORT_UASTC -// ------------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------------ // KTX2 -// ------------------------------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_KTX2 const uint8_t g_ktx2_file_identifier[12] = { 0xAB, 0x4B, 0x54, 0x58, 0x20, 0x32, 0x30, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A }; @@ -18854,8 +19394,8 @@ namespace basist m_key_values.clear(); memset((void *)&m_etc1s_header, 0, sizeof(m_etc1s_header)); m_etc1s_image_descs.clear(); - m_astc_6x6_intermediate_image_descs.clear(); - + m_slice_offset_len_descs.clear(); + m_format = basist::basis_tex_format::cETC1S; m_dfd_color_model = 0; @@ -18867,14 +19407,19 @@ namespace basist m_dfd_chan1 = KTX2_DF_CHANNEL_UASTC_RGB; m_etc1s_transcoder.clear(); - + m_def_transcoder_state.clear(); - + m_has_alpha = false; m_is_video = false; m_ldr_hdr_upconversion_nit_multiplier = 0.0f; } + static bool is_vk_format_astc_ldr(uint32_t fmt) + { + return (fmt >= KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK) && (fmt <= KTX2_FORMAT_ASTC_12x12_SRGB_BLOCK); + } + bool ktx2_transcoder::init(const void* pData, uint32_t data_size) { clear(); @@ -18904,9 +19449,10 @@ namespace basist memcpy((void *)&m_header, pData, sizeof(m_header)); // Check for supported VK formats. We may also need to parse the DFD. - if ((m_header.m_vk_format != KTX2_VK_FORMAT_UNDEFINED) && - (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK) && - (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK)) + if ((m_header.m_vk_format != KTX2_VK_FORMAT_UNDEFINED) && + (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK) && + (m_header.m_vk_format != basist::KTX2_FORMAT_ASTC_6x6_SFLOAT_BLOCK) && + !is_vk_format_astc_ldr(m_header.m_vk_format)) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: KTX2 file must be in ETC1S or UASTC LDR/HDR format\n"); return false; @@ -18943,7 +19489,7 @@ namespace basist return false; } } - + // 3.7 levelCount: "levelCount=0 is allowed, except for block-compressed formats" if (m_header.m_level_count < 1) { @@ -18966,14 +19512,6 @@ namespace basist if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) { -#if 0 - if (m_header.m_sgd_byte_length <= sizeof(ktx2_etc1s_global_data_header)) - { - BASISU_DEVEL_ERROR("ktx2_transcoder::init: Supercompression global data is too small\n"); - return false; - } -#endif - if (m_header.m_sgd_byte_offset.get_uint64() < sizeof(ktx2_header)) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: Supercompression global data offset is too low\n"); @@ -19002,7 +19540,7 @@ namespace basist } memcpy((void *)&m_levels[0], m_pData + sizeof(ktx2_header), level_index_size_in_bytes); - + // Sanity check the level offsets and byte sizes for (uint32_t i = 0; i < m_levels.size(); i++) { @@ -19022,9 +19560,9 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid level offset and/or length\n"); return false; } - + const uint64_t MAX_SANE_LEVEL_UNCOMP_SIZE = 2048ULL * 1024ULL * 1024ULL; - + if (m_levels[i].m_uncompressed_byte_length.get_uint64() >= MAX_SANE_LEVEL_UNCOMP_SIZE) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid level offset (too large)\n"); @@ -19061,7 +19599,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid DFD offset and/or length\n"); return false; } - + const uint8_t* pDFD = m_pData + m_header.m_dfd_byte_offset; if (!m_dfd.try_resize(m_header.m_dfd_byte_length)) @@ -19071,17 +19609,16 @@ namespace basist } memcpy(m_dfd.data(), pDFD, m_header.m_dfd_byte_length); - - // This is all hard coded for only ETC1S and UASTC. + uint32_t dfd_total_size = basisu::read_le_dword(pDFD); - + // 3.10.3: Sanity check if (dfd_total_size != m_header.m_dfd_byte_length) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: DFD size validation failed (1)\n"); return false; } - + // 3.10.3: More sanity checking if (m_header.m_kvd_byte_length) { @@ -19094,12 +19631,16 @@ namespace basist const uint32_t dfd_bits = basisu::read_le_dword(pDFD + 3 * sizeof(uint32_t)); const uint32_t sample_channel0 = basisu::read_le_dword(pDFD + 7 * sizeof(uint32_t)); - + const uint32_t texel_block_dimensions = basisu::read_le_dword(pDFD + 4 * sizeof(uint32_t)); + m_dfd_color_model = dfd_bits & 255; m_dfd_color_prims = (ktx2_df_color_primaries)((dfd_bits >> 8) & 255); m_dfd_transfer_func = (dfd_bits >> 16) & 255; m_dfd_flags = (dfd_bits >> 24) & 255; + const uint32_t block_width = (texel_block_dimensions & 0xFF) + 1; + const uint32_t block_height = ((texel_block_dimensions >> 8) & 0xFF) + 1; + // See 3.10.1.Restrictions if ((m_dfd_transfer_func != KTX2_KHR_DF_TRANSFER_LINEAR) && (m_dfd_transfer_func != KTX2_KHR_DF_TRANSFER_SRGB)) { @@ -19107,7 +19648,59 @@ namespace basist return false; } - if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ETC1S) + if (is_vk_format_astc_ldr(m_header.m_vk_format)) + { + // ASTC LDR 4x4-12x12 + // We usually read the DFD and decide the format from there. This decides off the VK format. + if (m_dfd_color_model != KTX2_KDF_DF_MODEL_ASTC) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid DFD color model (expected ASTC)\n"); + return false; + } + + uint32_t vk_fmt = m_header.m_vk_format; + const bool is_srgb_fmt = (vk_fmt & 1) == 0; + + if (is_srgb_fmt) + vk_fmt--; + + switch (vk_fmt) + { + case KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_4x4; break; + case KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_5x4; break; + case KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_5x5; break; + case KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_6x5; break; + case KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_6x6; break; + case KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x5; break; + case KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x6; break; + case KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_8x8; break; + case KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x5; break; + case KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x6; break; + case KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x8; break; + case KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_10x10; break; + case KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_12x10; break; + case KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK: m_format = basis_tex_format::cASTC_LDR_12x12; break; + default: + assert(0); + return false; + } + + // Sanity check the vkformat's astc block size vs. the DFD's. + uint32_t actual_block_width = 0, actual_block_height = 0; + get_basis_tex_format_block_size(m_format, actual_block_width, actual_block_height); + if ((actual_block_width != block_width) || (actual_block_height != block_height)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: vkFormat's ASTC block size is not in sync with the DFD's block dimensions\n"); + return false; + } + + m_dfd_samples = 1; + m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); + + // We're assuming "DATA" means RGBA so it has alpha. + m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); + } + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ETC1S) { if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) { @@ -19116,11 +19709,11 @@ namespace basist } m_format = basist::basis_tex_format::cETC1S; - + // 3.10.2: "Whether the image has 1 or 2 slices can be determined from the DFD's sample count." // If m_has_alpha is true it may be 2-channel RRRG or 4-channel RGBA, but we let the caller deal with that. m_has_alpha = (m_header.m_dfd_byte_length == 60); - + m_dfd_samples = m_has_alpha ? 2 : 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); @@ -19138,11 +19731,11 @@ namespace basist return false; } - m_format = basist::basis_tex_format::cUASTC4x4; + m_format = basist::basis_tex_format::cUASTC_LDR_4x4; m_dfd_samples = 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); - + // We're assuming "DATA" means RGBA so it has alpha. m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); } @@ -19180,7 +19773,7 @@ namespace basist m_has_alpha = false; } - else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE) + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE) { // Custom variable block size ASTC HDR 6x6 texture data. if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) @@ -19189,20 +19782,59 @@ namespace basist return false; } - m_format = basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE; + m_format = basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE; m_dfd_samples = 1; m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); m_has_alpha = false; } + else if (m_dfd_color_model == KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE) + { + if (m_header.m_vk_format != basist::KTX2_VK_FORMAT_UNDEFINED) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Invalid header vkFormat\n"); + return false; + } + + // Extract ASTC block dimensions from texel_block_dimensions, validate, select basis_tex_format. + m_format = basist::basis_tex_format::cETC1S; // bogus value to start + +#define BUT_BLOCK_SIZE(x, y, t) if ((block_width == (x)) && (block_height == (y))) { m_format = (t); } + BUT_BLOCK_SIZE(4, 4, basis_tex_format::cXUASTC_LDR_4x4); + BUT_BLOCK_SIZE(5, 4, basis_tex_format::cXUASTC_LDR_5x4); + BUT_BLOCK_SIZE(5, 5, basis_tex_format::cXUASTC_LDR_5x5); + BUT_BLOCK_SIZE(6, 5, basis_tex_format::cXUASTC_LDR_6x5); + BUT_BLOCK_SIZE(6, 6, basis_tex_format::cXUASTC_LDR_6x6); + BUT_BLOCK_SIZE(8, 5, basis_tex_format::cXUASTC_LDR_8x5); + BUT_BLOCK_SIZE(8, 6, basis_tex_format::cXUASTC_LDR_8x6); + BUT_BLOCK_SIZE(10, 5, basis_tex_format::cXUASTC_LDR_10x5); + BUT_BLOCK_SIZE(10, 6, basis_tex_format::cXUASTC_LDR_10x6); + BUT_BLOCK_SIZE(8, 8, basis_tex_format::cXUASTC_LDR_8x8); + BUT_BLOCK_SIZE(10, 8, basis_tex_format::cXUASTC_LDR_10x8); + BUT_BLOCK_SIZE(10, 10, basis_tex_format::cXUASTC_LDR_10x10); + BUT_BLOCK_SIZE(12, 10, basis_tex_format::cXUASTC_LDR_12x10); + BUT_BLOCK_SIZE(12, 12, basis_tex_format::cXUASTC_LDR_12x12); +#undef BUT_BLOCK_SIZE + + if (m_format == basist::basis_tex_format::cETC1S) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::init: Unsupported XUASTC LDR block dimensions (not valid ASTC)\n"); + return false; + } + + m_dfd_samples = 1; + m_dfd_chan0 = (ktx2_df_channel_id)((sample_channel0 >> 24) & 15); + + m_has_alpha = (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RGBA) || (m_dfd_chan0 == KTX2_DF_CHANNEL_UASTC_RRRG); + } else { // Unsupported DFD color model. BASISU_DEVEL_ERROR("ktx2_transcoder::init: Unsupported DFD color model\n"); return false; } - + if (!read_key_values()) { BASISU_DEVEL_ERROR("ktx2_transcoder::init: read_key_values() failed\n"); @@ -19261,7 +19893,7 @@ namespace basist return nullptr; } - + bool ktx2_transcoder::start_transcoding() { if (!m_pData) @@ -19270,14 +19902,14 @@ namespace basist return false; } - if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) + if (m_header.m_supercompression_scheme == KTX2_SS_BASISLZ) { if (m_format == basis_tex_format::cETC1S) { // Check if we've already decompressed the ETC1S global data. If so don't unpack it again. if (!m_etc1s_transcoder.get_endpoints().empty()) return true; - + if (!decompress_etc1s_global_data()) { BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: decompress_etc1s_global_data() failed\n"); @@ -19301,14 +19933,15 @@ namespace basist } } } - else if (m_format == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if ( (m_format == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) || basis_tex_format_is_xuastc_ldr(m_format) ) { - if (m_astc_6x6_intermediate_image_descs.size()) + // UASTC HDR 6x6 and XUASTC LDR 4x4-12x12 require an array of slice offset/len structs to determine where the compressed data starts for each independent compressed texture slice. + if (m_slice_offset_len_descs.size()) return true; - if (!read_astc_6x6_hdr_intermediate_global_data()) + if (!read_slice_offset_len_global_data()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: read_astc_6x6_hdr_intermediate_global_data() failed\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::start_transcoding: read_slice_offset_len_global_data() failed\n"); return false; } } @@ -19356,7 +19989,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::get_image_level_info: layer_index >= maximum(m_header.m_layer_count, 1)\n"); return false; } - + const uint32_t level_width = basisu::maximum(m_header.m_pixel_width >> level_index, 1); const uint32_t level_height = basisu::maximum(m_header.m_pixel_height >> level_index, 1); @@ -19380,7 +20013,7 @@ namespace basist level_info.m_total_blocks = num_blocks_x * num_blocks_y; level_info.m_alpha_flag = m_has_alpha; level_info.m_iframe_flag = false; - + if (m_etc1s_image_descs.size()) { const uint32_t etc1s_image_index = @@ -19393,9 +20026,9 @@ namespace basist return true; } - + bool ktx2_transcoder::transcode_image_level( - uint32_t level_index, uint32_t layer_index, uint32_t face_index, + uint32_t level_index, uint32_t layer_index, uint32_t face_index, void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, basist::transcoder_texture_format fmt, uint32_t decode_flags, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, int channel0, int channel1, @@ -19403,16 +20036,16 @@ namespace basist { if (!m_pData) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Must call init() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Must call init() first\n"); return false; } if (!pState) pState = &m_def_transcoder_state; - + if (level_index >= m_levels.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: level_index >= m_levels.size()\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: level_index >= m_levels.size()\n"); return false; } @@ -19420,34 +20053,34 @@ namespace basist { if (face_index >= 6) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: face_index >= 6\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: face_index >= 6\n"); return false; } } else if (face_index != 0) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: face_index != 0\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: face_index != 0\n"); return false; } if (layer_index >= basisu::maximum(m_header.m_layer_count, 1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: layer_index >= maximum(m_header.m_layer_count, 1)\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: layer_index >= maximum(m_header.m_layer_count, 1)\n"); return false; } const uint8_t* pComp_level_data = m_pData + m_levels[level_index].m_byte_offset.get_uint64(); uint64_t comp_level_data_size = m_levels[level_index].m_byte_length.get_uint64(); - + const uint8_t* pUncomp_level_data = pComp_level_data; uint64_t uncomp_level_data_size = comp_level_data_size; if (uncomp_level_data_size > UINT32_MAX) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_level_data_size > UINT32_MAX\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_level_data_size > UINT32_MAX\n"); return false; } - + if (m_header.m_supercompression_scheme == KTX2_SS_ZSTANDARD) { // Check if we've already decompressed this level's supercompressed data. @@ -19456,7 +20089,7 @@ namespace basist // Uncompress the entire level's supercompressed data. if (!decompress_level_data(level_index, pState->m_level_uncomp_data)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: decompress_level_data() failed\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: decompress_level_data() failed\n"); return false; } pState->m_uncomp_data_level_index = level_index; @@ -19465,18 +20098,19 @@ namespace basist pUncomp_level_data = pState->m_level_uncomp_data.data(); uncomp_level_data_size = pState->m_level_uncomp_data.size(); } - + const uint32_t level_width = basisu::maximum(m_header.m_pixel_width >> level_index, 1); const uint32_t level_height = basisu::maximum(m_header.m_pixel_height >> level_index, 1); const uint32_t num_blocks4_x = (level_width + 3) >> 2; const uint32_t num_blocks4_y = (level_height + 3) >> 2; - + if (m_format == basist::basis_tex_format::cETC1S) { + // ETC1S // Ensure start_transcoding() was called. if (m_etc1s_transcoder.get_endpoints().empty()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: must call start_transcoding() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); return false; } @@ -19484,12 +20118,11 @@ namespace basist (level_index * basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count) + layer_index * m_header.m_face_count + face_index; - + // Sanity check if (etc1s_image_index >= m_etc1s_image_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: etc1s_image_index >= m_etc1s_image_descs.size()\n"); - assert(0); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: etc1s_image_index >= m_etc1s_image_descs.size()\n"); return false; } @@ -19504,15 +20137,16 @@ namespace basist decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, &pState->m_transcoder_state, output_rows_in_pixels)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ETC1S transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ETC1S transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } - else if (m_format == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (m_format == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { - if (!m_astc_6x6_intermediate_image_descs.size()) + // UASTC HDR 6x6 + if (!m_slice_offset_len_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: must call start_transcoding() first\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); return false; } @@ -19525,102 +20159,219 @@ namespace basist face_index; // Sanity check - if (image_index >= m_astc_6x6_intermediate_image_descs.size()) + if (image_index >= m_slice_offset_len_descs.size()) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Invalid image_index\n"); - assert(0); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Invalid image_index\n"); return false; } - const ktx2_astc_hdr_6x6_intermediate_image_desc& image_desc = m_astc_6x6_intermediate_image_descs[image_index]; - + const ktx2_slice_offset_len_desc& image_desc = m_slice_offset_len_descs[image_index]; + if (!m_astc_hdr_6x6_intermediate_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, m_pData, m_data_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, - m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_rgb_slice_byte_offset, image_desc.m_rgb_slice_byte_length, + m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_slice_byte_offset, image_desc.m_slice_byte_length, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } else if (m_format == basist::basis_tex_format::cASTC_HDR_6x6) { + // plain ASTC HDR 6x6 const uint32_t num_blocks6_x = (level_width + 5) / 6; const uint32_t num_blocks6_y = (level_height + 5) / 6; // Compute length and offset to uncompressed 2D UASTC texture data, given the face/layer indices. assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); - const uint32_t total_2D_image_size = num_blocks6_x * num_blocks6_y * sizeof(astc_helpers::astc_block); - const uint32_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + const uint64_t total_2D_image_size = (uint64_t)num_blocks6_x * num_blocks6_y * sizeof(astc_helpers::astc_block); + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } // Sanity checks if (uncomp_ofs >= uncomp_level_data_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_ofs >= total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); return false; } if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); return false; } + assert(total_2D_image_size <= UINT32_MAX); + if (!m_astc_hdr_6x6_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, - (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, + (const uint8_t*)pUncomp_level_data + (size_t)uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks6_x, num_blocks6_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } - else if ((m_format == basist::basis_tex_format::cUASTC4x4) || - (m_format == basist::basis_tex_format::cUASTC_HDR_4x4)) + else if (basis_tex_format_is_astc_ldr(m_format)) { - // Compute length and offset to uncompressed 2D UASTC texture data, given the face/layer indices. - assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); - const uint32_t total_2D_image_size = num_blocks4_x * num_blocks4_y * KTX2_UASTC_BLOCK_SIZE; + // ASTC LDR 4x4-12x12 + const uint32_t block_width = get_block_width(), block_height = get_block_height(); - const uint32_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + const uint32_t num_blocks_x = (level_width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (level_height + block_height - 1) / block_height; + + //assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); + if (uncomp_level_data_size != m_levels[level_index].m_uncompressed_byte_length.get_uint64()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: m_uncompressed_byte_length is invalid\n"); + return false; + } + + const uint64_t total_2D_image_size = (uint64_t)num_blocks_x * num_blocks_y * sizeof(astc_helpers::astc_block); + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } // Sanity checks if (uncomp_ofs >= uncomp_level_data_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: uncomp_ofs >= total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); return false; } if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); return false; } + assert(total_2D_image_size <= UINT32_MAX); + + // if the header's vkformat is odd, it's linear, even is sRGB + const bool uses_astc_src_decode_profile = ((uint32_t)m_header.m_vk_format & 1) == 0; + + if (!m_xuastc_ldr_transcoder.transcode_image(m_format, uses_astc_src_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + (const uint8_t*)pUncomp_level_data + (size_t)uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks_x, num_blocks_y, level_width, level_height, level_index, + 0, (uint32_t)total_2D_image_size, + decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: ASTC 6x6 HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + return false; + } + + } + else if (basis_tex_format_is_xuastc_ldr(m_format)) + { + // XUASTC LDR 4x4-12x12 + if (!m_slice_offset_len_descs.size()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: must call start_transcoding() first\n"); + return false; + } + + const uint32_t block_width = get_block_width(), block_height = get_block_height(); + + const uint32_t num_blocks_x = (level_width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (level_height + block_height - 1) / block_height; + + const uint32_t image_index = + (level_index * basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count) + + layer_index * m_header.m_face_count + + face_index; + + // Sanity check + if (image_index >= m_slice_offset_len_descs.size()) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Invalid image_index\n"); + return false; + } + + const ktx2_slice_offset_len_desc& image_desc = m_slice_offset_len_descs[image_index]; + + // XUASTC LDR has its own tiny header at the start of the compressed data with this profile bit, so it'll use that for decoding if needed. + bool uses_astc_src_decode_profile = true; + + if (!m_xuastc_ldr_transcoder.transcode_image(m_format, uses_astc_src_decode_profile, fmt, + pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, + m_pData, m_data_size, num_blocks_x, num_blocks_y, level_width, level_height, level_index, + m_levels[level_index].m_byte_offset.get_uint64() + image_desc.m_slice_byte_offset, image_desc.m_slice_byte_length, + decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: XUASTC LDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + return false; + } + + } + else if ((m_format == basist::basis_tex_format::cUASTC_LDR_4x4) || + (m_format == basist::basis_tex_format::cUASTC_HDR_4x4)) + { + // UASTC LDR 4x4 and UASTC HDR 4x4 + + // Compute length and offset to uncompressed 2D UASTC texture data, given the face/layer indices. + assert(uncomp_level_data_size == m_levels[level_index].m_uncompressed_byte_length.get_uint64()); + const uint64_t total_2D_image_size = (uint64_t)num_blocks4_x * num_blocks4_y * KTX2_UASTC_BLOCK_SIZE; + + const uint64_t uncomp_ofs = (layer_index * m_header.m_face_count + face_index) * total_2D_image_size; + + if ((total_2D_image_size > UINT32_MAX) || ((size_t)uncomp_ofs != uncomp_ofs)) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: size too large\n"); + return false; + } + + // Sanity checks + if (uncomp_ofs >= uncomp_level_data_size) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: uncomp_ofs >= total_2D_image_size\n"); + return false; + } + + if ((uncomp_level_data_size - uncomp_ofs) < total_2D_image_size) + { + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: (uncomp_level_data_size - uncomp_ofs) < total_2D_image_size\n"); + return false; + } + + assert(total_2D_image_size <= UINT32_MAX); + if (m_format == basist::basis_tex_format::cUASTC_HDR_4x4) { + // UASTC HDR 4x4 if (!m_uastc_hdr_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks4_x, num_blocks4_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: UASTC HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: UASTC HDR transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } else { - if (!m_uastc_transcoder.transcode_image(fmt, + // UASTC LDR 4x4 + if (!m_uastc_ldr_transcoder.transcode_image(fmt, pOutput_blocks, output_blocks_buf_size_in_blocks_or_pixels, (const uint8_t*)pUncomp_level_data + uncomp_ofs, (uint32_t)total_2D_image_size, num_blocks4_x, num_blocks4_y, level_width, level_height, level_index, 0, (uint32_t)total_2D_image_size, decode_flags, m_has_alpha, m_is_video, output_row_pitch_in_blocks_or_pixels, nullptr, output_rows_in_pixels, channel0, channel1)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: UASTC transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: UASTC transcode_image() failed, this is either a bug or the file is corrupted/invalid\n"); return false; } } @@ -19628,19 +20379,19 @@ namespace basist else { // Shouldn't get here. - BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_2D: Internal error\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::transcode_image_level: Internal error\n"); assert(0); return false; } return true; } - + bool ktx2_transcoder::decompress_level_data(uint32_t level_index, basisu::uint8_vec& uncomp_data) { const uint8_t* pComp_data = m_levels[level_index].m_byte_offset.get_uint64() + m_pData; const uint64_t comp_size = m_levels[level_index].m_byte_length.get_uint64(); - + const uint64_t uncomp_size = m_levels[level_index].m_uncompressed_byte_length.get_uint64(); if (((size_t)comp_size) != comp_size) @@ -19659,7 +20410,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_level_data: Out of memory\n"); return false; } - + if (m_header.m_supercompression_scheme == KTX2_SS_ZSTANDARD) { #if BASISD_SUPPORT_KTX2_ZSTD @@ -19683,38 +20434,38 @@ namespace basist return true; } - bool ktx2_transcoder::read_astc_6x6_hdr_intermediate_global_data() + bool ktx2_transcoder::read_slice_offset_len_global_data() { const uint32_t image_count = basisu::maximum(m_header.m_layer_count, 1) * m_header.m_face_count * m_header.m_level_count; assert(image_count); const uint8_t* pSrc = m_pData + m_header.m_sgd_byte_offset.get_uint64(); - if (m_header.m_sgd_byte_length.get_uint64() != image_count * sizeof(ktx2_astc_hdr_6x6_intermediate_image_desc)) + if (m_header.m_sgd_byte_length.get_uint64() != image_count * sizeof(ktx2_slice_offset_len_desc)) { - BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_astc_6x6_hdr_intermediate_global_data: Invalid global data length\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::read_slice_offset_len_global_data: Invalid global data length\n"); return false; } - m_astc_6x6_intermediate_image_descs.resize(image_count); - - memcpy((void *)m_astc_6x6_intermediate_image_descs.data(), pSrc, sizeof(ktx2_astc_hdr_6x6_intermediate_image_desc) * image_count); + m_slice_offset_len_descs.resize(image_count); + + memcpy((void *)m_slice_offset_len_descs.data(), pSrc, sizeof(ktx2_slice_offset_len_desc) * image_count); // Sanity check the image descs for (uint32_t i = 0; i < image_count; i++) { // transcode_image() will validate the slice offsets/lengths before transcoding. - if (!m_astc_6x6_intermediate_image_descs[i].m_rgb_slice_byte_length) + if (!m_slice_offset_len_descs[i].m_slice_byte_length) { - BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_astc_6x6_hdr_intermediate_global_data: image descs sanity check failed (1)\n"); + BASISU_DEVEL_ERROR("ktx2_transcoder::read_slice_offset_len_global_data: image descs sanity check failed (1)\n"); return false; } } return true; } - + bool ktx2_transcoder::decompress_etc1s_global_data() { // Note: we don't actually support 3D textures in here yet @@ -19753,13 +20504,13 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: SGD byte length is too small, file is invalid or corrupted\n"); return false; } - + if (!m_etc1s_image_descs.try_resize(image_count)) { BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: Out of memory\n"); return false; } - + memcpy((void *)m_etc1s_image_descs.data(), pSrc, sizeof(ktx2_etc1s_image_desc) * image_count); pSrc += sizeof(ktx2_etc1s_image_desc) * image_count; @@ -19793,7 +20544,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: decode_tables() failed, file is invalid or corrupted\n"); return false; } - + if (!m_etc1s_transcoder.decode_palettes( m_etc1s_header.m_endpoint_count, pEndpoint_data, m_etc1s_header.m_endpoints_byte_length, m_etc1s_header.m_selector_count, pSelector_data, m_etc1s_header.m_selectors_byte_length)) @@ -19801,7 +20552,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::decompress_etc1s_global_data: decode_palettes() failed, file is likely corrupted\n"); return false; } - + return true; } @@ -19842,7 +20593,7 @@ namespace basist while (src_left > sizeof(uint32_t)) { uint32_t l = basisu::read_le_dword(pSrc); - + pSrc += sizeof(uint32_t); src_left -= sizeof(uint32_t); @@ -19863,7 +20614,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); return false; } - + basisu::uint8_vec& key_data = m_key_values.back().m_key; basisu::uint8_vec& value_data = m_key_values.back().m_value; @@ -19892,7 +20643,7 @@ namespace basist BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); return false; } - + if (!value_data.try_resize(l)) { BASISU_DEVEL_ERROR("ktx2_transcoder::read_key_values: Out of memory\n"); @@ -19928,7 +20679,7 @@ namespace basist return true; } - + #endif // BASISD_SUPPORT_KTX2 bool basisu_transcoder_supports_ktx2() @@ -19995,10 +20746,10 @@ namespace basist basist::half_float result = (basist::half_float)((s << 15) | (e << 10) | m); return result; } - + //------------------------------------------------------------------------------------------------ // HDR support - // + // // Originally from bc6h_enc.cpp // BC6H decoder fuzzed vs. DirectXTex's for unsigned/signed @@ -20027,7 +20778,7 @@ namespace basist const bc6h_bit_layout g_bc6h_bit_layouts[NUM_BC6H_MODES][MAX_BC6H_LAYOUT_INDEX] = { // comp_index, subset*2+lh_index, last_bit, first_bit - //------------------------ mode 0: 2 subsets, Weight bits: 46 bits, Endpoint bits: 75 bits (10.555, 10.555, 10.555), delta + //------------------------ mode 0: 2 subsets, Weight bits: 46 bits, Endpoint bits: 75 bits (10.555, 10.555, 10.555), delta { { 1, 2, 4, -1 }, { 2, 2, 4, -1 }, { 2, 3, 4, -1 }, { 0, 0, 9, 0 }, { 1, 0, 9, 0 }, { 2, 0, 9, 0 }, { 0, 1, 4, 0 }, { 1, 3, 4, -1 }, { 1, 2, 3, 0 }, { 1, 1, 4, 0 }, { 2, 3, 0, -1 }, { 1, 3, 3, 0 }, { 2, 1, 4, 0 }, { 2, 3, 1, -1 }, { 2, 2, 3, 0 }, { 0, 2, 4, 0 }, { 2, 3, 2, -1 }, { 0, 3, 4, 0 }, { 2, 3, 3, -1 }, { 3, -1, 4, 0 }, {-1, 0, 0, 0} }, @@ -20078,7 +20829,7 @@ namespace basist { { 0, 0, 9, 0 },{ 1, 0, 9, 0 },{ 2, 0, 9, 0 },{ 0, 1, 3, 0 },{ 0, 0, 10, 15 },{ 1, 1, 3, 0 },{ 1, 0, 10, 15 },{ 2, 1, 3, 0 },{ 2, 0, 10, 15 }, {-1, 0, 0, 0} } }; - // The same as the first 32 2-subset patterns in BC7. + // The same as the first 32 2-subset patterns in BC7. // Bit 7 is a flag indicating that the weight uses 1 less bit than usual. const uint8_t g_bc6h_2subset_patterns[TOTAL_BC6H_PARTITION_PATTERNS][4][4] = // [pat][y][x] { @@ -20102,7 +20853,7 @@ namespace basist const uint8_t g_bc6h_weight3[8] = { 0, 9, 18, 27, 37, 46, 55, 64 }; const uint8_t g_bc6h_weight4[16] = { 0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64 }; - + static inline void write_bits(uint64_t val, uint32_t num_bits, uint32_t& bit_pos, uint64_t& l, uint64_t& h) { assert((num_bits) && (num_bits < 64) && (bit_pos < 128)); @@ -20231,15 +20982,18 @@ namespace basist return unq; } #endif - + // 6,7,8,9,10,11,12 const uint32_t BC6H_BLOG_TAB_MIN = 6; const uint32_t BC6H_BLOG_TAB_MAX = 12; //const uint32_t BC6H_BLOG_TAB_NUM = BC6H_BLOG_TAB_MAX - BC6H_BLOG_TAB_MIN + 1; - + // Handles 16, or 6-12 bits. Others assert. static inline uint32_t half_to_blog_tab(half_float h, uint32_t num_bits) { + BASISU_NOTE_UNUSED(BC6H_BLOG_TAB_MIN); + BASISU_NOTE_UNUSED(BC6H_BLOG_TAB_MAX); + assert(h <= MAX_BC6H_HALF_FLOAT_AS_UINT); assert((num_bits == 16) || ((num_bits >= BC6H_BLOG_TAB_MIN) && (num_bits <= BC6H_BLOG_TAB_MAX))); @@ -20255,7 +21009,7 @@ namespace basist else { assert((num_bits >= BC6H_BLOG_TAB_MIN) && (num_bits <= BC6H_BLOG_TAB_MAX)); - + // Note: This used to be done using a table lookup, but it required ~224KB of tables. This isn't quite as accurate, but the error is very slight (+-1 half values as ints). return bc6h_half_to_blog(h, num_bits); } @@ -20385,7 +21139,7 @@ namespace basist log_blk.m_mode = mode; pack_bc6h_block(*pPacked_block, log_blk); - + return; } @@ -20785,13 +21539,13 @@ namespace basist half_float endpoints[3][2]; endpoints[0][0] = pColor[0]; endpoints[0][1] = pColor[0]; - + endpoints[1][0] = pColor[1]; endpoints[1][1] = pColor[1]; endpoints[2][0] = pColor[2]; endpoints[2][1] = pColor[2]; - + bc6h_enc_block_1subset_4bit_weights(pPacked_block, endpoints, weights); return true; @@ -20828,7 +21582,7 @@ namespace basist static inline int astc_hdr_sign_extend(int src, int num_src_bits) { - assert(basisu::in_range(num_src_bits, 2, 31)); + assert(basisu::is_in_range(num_src_bits, 2, 31)); const bool negative = (src & (1 << (num_src_bits - 1))) != 0; if (negative) @@ -21069,7 +21823,7 @@ namespace basist if (ohm & 0x12) vb1 |= (x3 << 7); const int shamt = (mode >> 1) ^ 3; - + va = (uint32_t)va << shamt; vb0 = (uint32_t)vb0 << shamt; vb1 = (uint32_t)vb1 << shamt; @@ -21139,7 +21893,7 @@ namespace basist { assert(g_astc_hdr_core_initialized); assert((best_blk.m_weight_ise_range >= 1) && (best_blk.m_weight_ise_range <= 8)); - + if (best_blk.m_weight_ise_range == 5) { // Use 3-bit BC6H weights which are a perfect match for 3-bit ASTC weights, but encode 1-subset as 2 equal subsets @@ -21221,7 +21975,7 @@ namespace basist assert(g_astc_hdr_core_initialized); assert(best_blk.m_num_partitions == 2); assert(common_part_index < basist::TOTAL_ASTC_BC6H_COMMON_PARTITIONS2); - + half_float bc6h_endpoints[2][3][2]; // [subset][comp][lh_index] // UASTC HDR checks @@ -21230,7 +21984,7 @@ namespace basist return false; if ((best_blk.m_color_endpoint_modes[0] != 7) && (best_blk.m_color_endpoint_modes[0] != 11)) return false; - + if (best_blk.m_color_endpoint_modes[0] == 7) { if (!(((best_blk.m_weight_ise_range == 1) && (best_blk.m_endpoint_ise_range == 20)) || @@ -21355,7 +22109,7 @@ namespace basist assert(0); return false; } - + if (log_blk.m_solid_color_flag_ldr) { // Don't support LDR solid colors. @@ -21371,7 +22125,7 @@ namespace basist // Only support 4x4 grid sizes if ((log_blk.m_grid_width != 4) || (log_blk.m_grid_height != 4)) return false; - + // Don't support dual plane encoding if (log_blk.m_dual_plane) return false; @@ -21379,11 +22133,11 @@ namespace basist if (log_blk.m_num_partitions == 1) { // Handle 1 partition (or subset) - + // UASTC HDR checks if ((log_blk.m_weight_ise_range < 1) || (log_blk.m_weight_ise_range > 8)) return false; - + int e[2][3]; bool success; @@ -21429,7 +22183,7 @@ namespace basist for (uint32_t i = 0; i < 2; i++) if (is_half_inf_or_nan(h_e[0][i]) || is_half_inf_or_nan(h_e[1][i]) || is_half_inf_or_nan(h_e[2][i])) return false; - + // Transcode to bc6h if (!transcode_bc6h_1subset(h_e, log_blk, dst_blk)) return false; @@ -21442,7 +22196,7 @@ namespace basist return false; assert(common_bc7_pat_index < (int)basist::TOTAL_ASTC_BC6H_COMMON_PARTITIONS2); - + if (!transcode_bc6h_2subsets(common_bc7_pat_index, log_blk, dst_blk)) return false; } @@ -21454,7 +22208,7 @@ namespace basist return true; } - + // ASTC 6x6 support namespace astc_6x6_hdr { @@ -21582,7 +22336,7 @@ namespace basist // 3x3 { false, 7, 3, 3, 3, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_8_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_8_LEVELS, 0, 0 }, - // 6x4 + // 6x4 { false, 7, 3, 6, 4, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, BASIST_HDR_6X6_LEVEL2, 0 }, { false, 7, 3, 4, 6, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, astc_helpers::BISE_64_LEVELS, astc_helpers::BISE_2_LEVELS, BASIST_HDR_6X6_LEVEL2, 0 }, @@ -21610,7 +22364,7 @@ namespace basist { false, 7, 3, 5, 4, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, 0, 0 }, { false, 7, 3, 4, 5, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, astc_helpers::BISE_40_LEVELS, astc_helpers::BISE_3_LEVELS, 0, 0 }, }; - + const reuse_xy_delta g_reuse_xy_deltas[NUM_REUSE_XY_DELTAS] = { { -1, 0 }, { -2, 0 }, { -3, 0 }, { -4, 0 }, @@ -21656,27 +22410,27 @@ namespace basist int bit = get_bit(src_val, src_bit); dst |= (bit << dst_bit); } - - // Valid for weight ISE ranges 12-192 levels. Preserves upper 2 or 3 bits post-quantization. - static uint8_t g_quantize_tables_preserve2[astc_helpers::TOTAL_ISE_RANGES - 1][256]; - static uint8_t g_quantize_tables_preserve3[astc_helpers::TOTAL_ISE_RANGES - 1][256]; + + // Valid for weight ISE ranges 6-192 or 8-192 levels. Preserves upper 2 or 3 bits post-quantization. + uint8_t g_quantize_tables_preserve2[21 - 1][256]; // astc_helpers::TOTAL_ISE_RANGES=21, valid for >= BISE_6_LEVELS + uint8_t g_quantize_tables_preserve3[21 - 1][256]; // valid for >= BISE_8_LEVELS const uint32_t g_part2_unique_index_to_seed[NUM_UNIQUE_PARTITIONS2] = { - 86, 959, 936, 476, 1007, 672, 447, 423, 488, 422, 273, 65, 267, 786, 585, 195, 108, 731, 878, 812, 264, 125, 868, 581, 258, 390, 549, 872, 661, 352, 645, 543, 988, - 906, 903, 616, 482, 529, 3, 286, 272, 303, 151, 504, 498, 260, 79, 66, 608, 769, 305, 610, 1014, 967, 835, 789, 7, 951, 691, 15, 763, 976, 438, 314, 601, 673, 177, - 252, 615, 436, 220, 899, 623, 433, 674, 278, 797, 107, 847, 114, 470, 760, 821, 490, 329, 945, 387, 471, 225, 172, 83, 418, 966, 439, 316, 247, 43, 343, 625, 798, - 1, 61, 73, 307, 136, 474, 42, 664, 1013, 249, 389, 227, 374, 121, 48, 538, 226, 309, 554, 802, 834, 335, 495, 10, 955, 461, 293, 508, 153, 101, 63, 139, 31, 687, - 132, 174, 324, 545, 289, 39, 178, 594, 963, 854, 222, 323, 998, 964, 598, 475, 720, 1019, 983, 91, 703, 614, 394, 612, 281, 207, 930, 758, 586, 128, 517, 426, 306, - 168, 713, 36, 458, 876, 368, 780, 5, 9, 214, 109, 553, 726, 175, 103, 753, 684, 44, 665, 53, 500, 367, 611, 119, 732, 639, 326, 203, 156, 686, 910, 255, 62, 392, 591, - 112, 88, 213, 19, 1022, 478, 90, 486, 799, 702, 730, 414, 99, 1008, 142, 886, 373, 216, 69, 393, 299, 648, 415, 822, 912, 110, 567, 550, 693, 2, 138, 59, 271, 562, 295, - 714, 719, 199, 893, 831, 1006, 662, 235, 262, 78, 51, 902, 298, 190, 169, 583, 347, 890, 958, 909, 49, 987, 696, 633, 480, 50, 764, 826, 1023, 1016, 437, 891, 774, 257, - 724, 791, 526, 593, 690, 638, 858, 895, 794, 995, 130, 87, 877, 819, 318, 649, 376, 211, 284, 937, 370, 688, 229, 994, 115, 842, 60, 521, 95, 694, 804, 146, 754, 487, 55, - 17, 770, 450, 223, 4, 137, 911, 236, 683, 523, 47, 181, 24, 270, 602, 736, 11, 355, 148, 351, 762, 1009, 16, 210, 619, 805, 874, 807, 887, 403, 999, 810, 27, 402, 551, 135, - 778, 33, 409, 993, 71, 363, 159, 183, 77, 596, 670, 380, 968, 811, 404, 348, 539, 158, 578, 196, 621, 68, 530, 193, 100, 167, 919, 353, 366, 327, 643, 948, 518, 756, 801, 558, - 28, 705, 116, 94, 898, 453, 622, 647, 231, 445, 652, 230, 191, 277, 292, 254, 198, 766, 386, 232, 29, 70, 942, 740, 291, 607, 411, 496, 839, 8, 675, 319, 742, 21, 547, 627, 716, - 663, 23, 914, 631, 595, 499, 685, 950, 510, 54, 587, 432, 45, 646, 25, 122, 947, 171, 862, 441, 808, 722, 14, 74, 658, 129, 266, 1001, 534, 395, 527, 250, 206, 237, 67, 897, 634, - 572, 569, 533, 37, 341, 89, 463, 419, 75, 134, 283, 943, 519, 362, 144, 681, 407, 954, 131, 455, 934, 46, 513, 339, 194, 361, 606, 852, 546, 655, 1015, 147, 506, 240, 56, 836, 76, + 86, 959, 936, 476, 1007, 672, 447, 423, 488, 422, 273, 65, 267, 786, 585, 195, 108, 731, 878, 812, 264, 125, 868, 581, 258, 390, 549, 872, 661, 352, 645, 543, 988, + 906, 903, 616, 482, 529, 3, 286, 272, 303, 151, 504, 498, 260, 79, 66, 608, 769, 305, 610, 1014, 967, 835, 789, 7, 951, 691, 15, 763, 976, 438, 314, 601, 673, 177, + 252, 615, 436, 220, 899, 623, 433, 674, 278, 797, 107, 847, 114, 470, 760, 821, 490, 329, 945, 387, 471, 225, 172, 83, 418, 966, 439, 316, 247, 43, 343, 625, 798, + 1, 61, 73, 307, 136, 474, 42, 664, 1013, 249, 389, 227, 374, 121, 48, 538, 226, 309, 554, 802, 834, 335, 495, 10, 955, 461, 293, 508, 153, 101, 63, 139, 31, 687, + 132, 174, 324, 545, 289, 39, 178, 594, 963, 854, 222, 323, 998, 964, 598, 475, 720, 1019, 983, 91, 703, 614, 394, 612, 281, 207, 930, 758, 586, 128, 517, 426, 306, + 168, 713, 36, 458, 876, 368, 780, 5, 9, 214, 109, 553, 726, 175, 103, 753, 684, 44, 665, 53, 500, 367, 611, 119, 732, 639, 326, 203, 156, 686, 910, 255, 62, 392, 591, + 112, 88, 213, 19, 1022, 478, 90, 486, 799, 702, 730, 414, 99, 1008, 142, 886, 373, 216, 69, 393, 299, 648, 415, 822, 912, 110, 567, 550, 693, 2, 138, 59, 271, 562, 295, + 714, 719, 199, 893, 831, 1006, 662, 235, 262, 78, 51, 902, 298, 190, 169, 583, 347, 890, 958, 909, 49, 987, 696, 633, 480, 50, 764, 826, 1023, 1016, 437, 891, 774, 257, + 724, 791, 526, 593, 690, 638, 858, 895, 794, 995, 130, 87, 877, 819, 318, 649, 376, 211, 284, 937, 370, 688, 229, 994, 115, 842, 60, 521, 95, 694, 804, 146, 754, 487, 55, + 17, 770, 450, 223, 4, 137, 911, 236, 683, 523, 47, 181, 24, 270, 602, 736, 11, 355, 148, 351, 762, 1009, 16, 210, 619, 805, 874, 807, 887, 403, 999, 810, 27, 402, 551, 135, + 778, 33, 409, 993, 71, 363, 159, 183, 77, 596, 670, 380, 968, 811, 404, 348, 539, 158, 578, 196, 621, 68, 530, 193, 100, 167, 919, 353, 366, 327, 643, 948, 518, 756, 801, 558, + 28, 705, 116, 94, 898, 453, 622, 647, 231, 445, 652, 230, 191, 277, 292, 254, 198, 766, 386, 232, 29, 70, 942, 740, 291, 607, 411, 496, 839, 8, 675, 319, 742, 21, 547, 627, 716, + 663, 23, 914, 631, 595, 499, 685, 950, 510, 54, 587, 432, 45, 646, 25, 122, 947, 171, 862, 441, 808, 722, 14, 74, 658, 129, 266, 1001, 534, 395, 527, 250, 206, 237, 67, 897, 634, + 572, 569, 533, 37, 341, 89, 463, 419, 75, 134, 283, 943, 519, 362, 144, 681, 407, 954, 131, 455, 934, 46, 513, 339, 194, 361, 606, 852, 546, 655, 1015, 147, 506, 240, 56, 836, 76, 98, 600, 430, 388, 980, 695, 817, 279, 58, 215, 149, 170, 531, 870, 18, 727, 154, 26, 938, 929, 302, 697, 452, 218, 700, 524, 828, 751, 869, 217, 440, 354 }; @@ -21707,7 +22461,9 @@ namespace basist static void init_quantize_tables() { - for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_12_LEVELS; ise_range--) + // 9/15/2025 changed lower range for LDR + // for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_12_LEVELS; ise_range--) + for (uint32_t ise_range = astc_helpers::BISE_192_LEVELS; ise_range >= astc_helpers::BISE_6_LEVELS; ise_range--) { const uint32_t num_levels = astc_helpers::get_ise_levels(ise_range); const auto& ise_to_val_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(ise_range).m_ISE_to_val; @@ -21739,6 +22495,7 @@ namespace basist g_quantize_tables_preserve2[ise_range][desired_val] = (uint8_t)best_ise_val; } + if (ise_range >= astc_helpers::BISE_8_LEVELS) { uint32_t best_err = UINT32_MAX; int best_ise_val = -1; @@ -21894,7 +22651,7 @@ namespace basist #endif } - void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk) + void copy_weight_grid(bool dual_plane, uint32_t grid_x, uint32_t grid_y, const uint8_t* transcode_weights, astc_helpers::log_astc_block& decomp_blk, bool orig_behavior) { assert(decomp_blk.m_weight_ise_range >= astc_helpers::BISE_2_LEVELS); assert(decomp_blk.m_weight_ise_range <= astc_helpers::BISE_32_LEVELS); @@ -21913,11 +22670,12 @@ namespace basist compute_upsample_weights(4, 4, 2, 2, weights); - for (uint32_t y = 0; y < 4; y++) + for (uint32_t dy = 0; dy < 4; dy++) { - for (uint32_t x = 0; x < 4; x++) + for (uint32_t dx = 0; dx < 4; dx++) { - const astc_helpers::weighted_sample& sample = weights[x + y * 4]; + const astc_helpers::weighted_sample& sample = weights[dx + dy * 4]; + const int sx = sample.m_src_x, sy = sample.m_src_y; uint32_t total_weight = 8; @@ -21928,7 +22686,19 @@ namespace basist if (!sample.m_weights[yo][xo]) continue; - total_weight += dequant_weight[transcode_weights[basisu::in_bounds((x + xo) + (y + yo) * grid_x, 0, grid_x * grid_y)]] * sample.m_weights[yo][xo]; + // 10/17/2025 - bugfix. Orig release would always sample the 1st or 2nd weight here. Minor issue - encoder would have detected it, hurting R-D performance a tiny bit but not encoding/decoding correctness. + // However, this fix does cause decoding divergence from original encodes. The divergence seems minor and can only happen at higher lambdas. + if (orig_behavior) + { + // Original, incorrect, but ultimately harmless behavior. + total_weight += dequant_weight[transcode_weights[basisu::is_in_bounds((dx + xo) + (dy + yo) * grid_x, 0, grid_x * grid_y)]] * sample.m_weights[yo][xo]; + } + else + { + // Correct behavior. + assert(basisu::is_in_bounds((sx + xo) + (sy + yo) * grid_x, 0, grid_x * grid_y)); + total_weight += dequant_weight[transcode_weights[(sx + xo) + (sy + yo) * grid_x]] * sample.m_weights[yo][xo]; + } } // x } // y @@ -21936,7 +22706,7 @@ namespace basist assert(total_weight <= 64); - decomp_blk.m_weights[x + y * 4] = quant_weight[total_weight]; + decomp_blk.m_weights[dx + dy * 4] = quant_weight[total_weight]; } } } @@ -22063,7 +22833,7 @@ namespace basist assert((cem == 7) || (cem == 11)); return (cem == 11) ? basist::NUM_MODE11_ENDPOINTS : basist::NUM_MODE7_ENDPOINTS; } - + const uint32_t g_bc6h_weights4[16] = { 0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64 }; #if 0 @@ -22147,7 +22917,7 @@ namespace basist float res = (float)fast_float_to_half_no_clamp_neg_nan_or_inf(fabsf(f)) * ((f < 0.0f) ? -1.0f : 1.0f); return res; } - + // Supports positive and denormals only. No NaN or Inf. static BASISU_FORCE_INLINE float fast_half_to_float_pos_not_inf_or_nan(basist::half_float h) { @@ -22188,7 +22958,7 @@ namespace basist int max_r, int max_g, int max_b, int64_t block_max_var) { BASISU_NOTE_UNUSED(block_max_var); - + float fmin_r = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_r); float fmin_g = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_g); float fmin_b = fast_half_to_float_pos_not_inf_or_nan((basist::half_float)min_b); @@ -22225,12 +22995,12 @@ namespace basist pWeights[i] = (uint8_t)basisu::clamp((int)(w * frr + lr), 0, 15); } } - + static double assign_weights_4( const vec3F* pFloat_pixels, const float* pPixel_scales, uint8_t* pWeights, int min_r, int min_g, int min_b, - int max_r, int max_g, int max_b, int64_t block_max_var, bool try_2subsets_flag, + int max_r, int max_g, int max_b, int64_t block_max_var, bool try_2subsets_flag, const fast_bc6h_params& params) { float cr[16], cg[16], cb[16]; @@ -22348,7 +23118,7 @@ namespace basist pWeights[i] = (uint8_t)best_idx; - // Giesen's MRSSE (Mean Relative Sum of Squared Errors). + // Giesen's MRSSE (Mean Relative Sum of Squared Errors). // Our ASTC HDR encoder uses slightly slower approx. MSLE, and it's too late/risky to eval the difference vs. MRSSE on the larger ASTC HDR blocks. float err = basisu::squaref(qr - cr[best_idx]) + basisu::squaref(qg - cg[best_idx]) + basisu::squaref(qb - cb[best_idx]); total_err += err * pPixel_scales[i]; @@ -22448,7 +23218,7 @@ namespace basist float best_error = basisu::squaref(subset_cr[subset][0] - qr) + basisu::squaref(subset_cg[subset][0] - qg) + basisu::squaref(subset_cb[subset][0] - qb); uint32_t best_idx = 0; - + for (uint32_t j = 1; j < 8; j++) { float e = basisu::squaref(subset_cr[subset][j] - qr) + basisu::squaref(subset_cg[subset][j] - qg) + basisu::squaref(subset_cb[subset][j] - qb); @@ -22521,7 +23291,7 @@ namespace basist static basist::vec4F g_bc6h_ls_weights_3[8]; static basist::vec4F g_bc6h_ls_weights_4[16]; - + const uint32_t BC6H_NUM_PATS = 32; static uint32_t g_bc6h_pats2[BC6H_NUM_PATS]; @@ -22587,7 +23357,7 @@ namespace basist static void bc6h_quant_endpoints( uint32_t min_hr, uint32_t min_hg, uint32_t min_hb, uint32_t max_hr, uint32_t max_hg, uint32_t max_hb, - uint32_t& min_r, uint32_t& min_g, uint32_t& min_b, uint32_t& max_r, uint32_t& max_g, uint32_t& max_b, + uint32_t& min_r, uint32_t& min_g, uint32_t& min_b, uint32_t& max_r, uint32_t& max_g, uint32_t& max_b, int bits) { min_r = basist::bc6h_half_to_blog((basist::half_float)min_hr, bits); @@ -22613,7 +23383,7 @@ namespace basist max_hb = bc6h_convert_to_half(bc6h_dequantize(max_bb, bits)); } - static BASISU_FORCE_INLINE int popcount32(uint32_t x) + static BASISU_FORCE_INLINE int popcount32(uint32_t x) { #if defined(__EMSCRIPTEN__) || defined(__clang__) || defined(__GNUC__) return __builtin_popcount(x); @@ -22621,7 +23391,7 @@ namespace basist return __popcnt(x); #else int count = 0; - while (x) + while (x) { x &= (x - 1); ++count; @@ -22634,23 +23404,23 @@ namespace basist { return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); } - + static void fast_encode_bc6h_2subsets_pattern( uint32_t best_pat_index, uint32_t best_pat_bits, const basist::half_float* pPixels, const vec3F* pFloat_pixels, const float* pPixel_scales, double& cur_error, basist::bc6h_logical_block& log_blk, int64_t block_max_var, - int mean_r, int mean_g, int mean_b, + int mean_r, int mean_g, int mean_b, const fast_bc6h_params& params) { BASISU_NOTE_UNUSED(block_max_var); - + uint32_t subset_means[2][3] = { { 0 } }; for (uint32_t i = 0; i < 16; i++) { const uint32_t subset_index = (best_pat_bits >> i) & 1; const uint32_t r = pPixels[i * 3 + 0], g = pPixels[i * 3 + 1], b = pPixels[i * 3 + 2]; - + subset_means[subset_index][0] += r; subset_means[subset_index][1] += g; subset_means[subset_index][2] += b; @@ -22703,7 +23473,7 @@ namespace basist subset_axis[subset_index].set(axis_r, axis_g, axis_b); } // s - + float subset_min_dot[2] = { basisu::BIG_FLOAT_VAL, basisu::BIG_FLOAT_VAL }; float subset_max_dot[2] = { -basisu::BIG_FLOAT_VAL, -basisu::BIG_FLOAT_VAL }; int subset_min_idx[2] = { 0 }, subset_max_idx[2] = { 0 }; @@ -22820,7 +23590,7 @@ namespace basist if (params.m_num_diff_endpoint_modes_to_try) { // ordered from largest base bits to least - static const int s_bc6h_mode_order2[2] = { 5, 1 }; + static const int s_bc6h_mode_order2[2] = { 5, 1 }; static const int s_bc6h_mode_order4[4] = { 0, 5, 7, 1 }; static const int s_bc6h_mode_order9[9] = { 2, 3, 4, 0, 5, 6, 7, 8, 1 }; @@ -22848,7 +23618,7 @@ namespace basist BASISU_NOTE_UNUSED(base_bitmask); const uint32_t num_delta_bits[3] = { g_bc6h_mode_sig_bits[mode][1], g_bc6h_mode_sig_bits[mode][2], g_bc6h_mode_sig_bits[mode][3] }; - const int delta_bitmasks[3] = { (1 << num_delta_bits[0]) - 1, (1 << num_delta_bits[1]) - 1, (1 << num_delta_bits[2]) - 1 }; + //const int delta_bitmasks[3] = { (1 << num_delta_bits[0]) - 1, (1 << num_delta_bits[1]) - 1, (1 << num_delta_bits[2]) - 1 }; for (uint32_t subset_index = 0; subset_index < 2; subset_index++) { @@ -22928,10 +23698,10 @@ namespace basist trial_log_blk.m_mode = bc6h_mode_index; trial_log_blk.m_partition_pattern = best_pat_index; - + memcpy(trial_log_blk.m_endpoints, abs_blog_endpoints, sizeof(trial_log_blk.m_endpoints)); memcpy(trial_log_blk.m_weights, trial_weights, 16); - + if (trial_log_blk.m_weights[0] & 4) { for (uint32_t c = 0; c < 3; c++) @@ -22958,7 +23728,7 @@ namespace basist trial_log_blk.m_weights[i] = 7 - trial_log_blk.m_weights[i]; } } - + if (bc6h_mode_index != BC6H_2SUBSET_ABS_ENDPOINT_MODE) { const uint32_t num_delta_bits[3] = { g_bc6h_mode_sig_bits[bc6h_mode_index][1], g_bc6h_mode_sig_bits[bc6h_mode_index][2], g_bc6h_mode_sig_bits[bc6h_mode_index][3] }; @@ -22999,7 +23769,7 @@ namespace basist const basist::half_float* pPixels, const vec3F* pFloat_pixels, const float* pPixel_scales, double& cur_error, basist::bc6h_logical_block& log_blk, int64_t block_max_var, - int mean_r, int mean_g, int mean_b, float block_axis_r, float block_axis_g, float block_axis_b, + int mean_r, int mean_g, int mean_b, float block_axis_r, float block_axis_g, float block_axis_b, const fast_bc6h_params& params) { assert((params.m_max_2subset_pats_to_try > 0) && (params.m_max_2subset_pats_to_try <= BC6H_NUM_PATS)); @@ -23020,7 +23790,7 @@ namespace basist } return; } - + uint32_t desired_pat_bits = 0; for (uint32_t i = 0; i < 16; i++) { @@ -23098,13 +23868,13 @@ namespace basist uint32_t omin_r = UINT32_MAX, omin_g = UINT32_MAX, omin_b = UINT32_MAX; uint32_t omax_r = 0, omax_g = 0, omax_b = 0; uint32_t total_r = 0, total_g = 0, total_b = 0; - + for (uint32_t i = 0; i < 16; i++) { uint32_t r = pPixels[i * 3 + 0]; uint32_t g = pPixels[i * 3 + 1]; uint32_t b = pPixels[i * 3 + 2]; - + total_r += r; total_g += g; total_b += b; @@ -23129,13 +23899,13 @@ namespace basist log_blk.m_endpoints[2][0] = basist::bc6h_half_to_blog16((basist::half_float)omin_b); log_blk.m_endpoints[2][1] = 0; - + log_blk.m_mode = 13; pack_bc6h_block(*pBlock, log_blk); return; } - + uint32_t min_r, min_g, min_b, max_r, max_g, max_b; int mean_r = (total_r + 8) / 16; @@ -23157,9 +23927,9 @@ namespace basist icov[4] += g * b; icov[5] += b * b; } - + int64_t block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 - + if (block_max_var < (FAST_BC6H_STD_DEV_THRESH * FAST_BC6H_STD_DEV_THRESH * 16)) { // Simple block @@ -23254,7 +24024,7 @@ namespace basist uint32_t min_idx = 0, max_idx = 0; float min_dot = basisu::BIG_FLOAT_VAL, max_dot = -basisu::BIG_FLOAT_VAL; - + for (uint32_t i = 0; i < 16; i++) { float r = (float)pPixels[i * 3 + 0]; @@ -23295,7 +24065,7 @@ namespace basist bc6h_quant_dequant_endpoints(min_r, min_g, min_b, max_r, max_g, max_b, 10); cur_err = assign_weights_4(float_pixels, pixel_scales, log_blk.m_weights, min_r, min_g, min_b, max_r, max_g, max_b, block_max_var, try_2subsets, params); - + const uint32_t MAX_LS_PASSES = params.m_hq_ls ? 2 : 1; for (uint32_t pass = 0; pass < MAX_LS_PASSES; pass++) { @@ -23371,7 +24141,7 @@ namespace basist min_b = trial_min_b; max_b = trial_max_b; - + memcpy(log_blk.m_weights, trial_weights, 16); } else @@ -23413,7 +24183,7 @@ namespace basist std::swap(log_blk.m_endpoints[c][0], log_blk.m_endpoints[c][1]); } } - + if ((params.m_max_2subset_pats_to_try > 0) && ((try_2subsets) && (block_max_var > (FAST_BC6H_COMPLEX_STD_DEV_THRESH * FAST_BC6H_COMPLEX_STD_DEV_THRESH * 16)))) { fast_encode_bc6h_2subsets(pPixels, float_pixels, pixel_scales, cur_err, log_blk, block_max_var, mean_r, mean_g, mean_b, axis_r, axis_g, axis_b, params); @@ -23439,7 +24209,12 @@ namespace basist if (!decoder.init(pComp_data, comp_data_size)) return false; - if (decoder.get_bits(16) != 0xABCD) + bool orig_behavior = false; + + uint32_t hdr_sig = decoder.get_bits(16); + if (hdr_sig == UASTC_6x6_HDR_SIG0) + orig_behavior = true; + else if (hdr_sig != UASTC_6x6_HDR_SIG1) return false; width = decoder.get_bits(16); @@ -23621,7 +24396,7 @@ namespace basist uint8_t transcode_weights[MAX_BLOCK_W * MAX_BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, log_blk.m_weight_ise_range, transcode_weights, decomp_blk.m_weight_ise_range); - copy_weight_grid(log_blk.m_dual_plane, log_blk.m_grid_width, log_blk.m_grid_height, transcode_weights, decomp_blk); + copy_weight_grid(log_blk.m_dual_plane, log_blk.m_grid_width, log_blk.m_grid_height, transcode_weights, decomp_blk, orig_behavior); #else assert(log_blk.m_user_mode < TOTAL_BLOCK_MODE_DECS); const block_mode_desc& bmd = g_block_mode_descs[(uint32_t)log_blk.m_user_mode]; @@ -23655,7 +24430,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, log_blk.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); #endif status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23741,7 +24516,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23839,7 +24614,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23923,7 +24698,7 @@ namespace basist uint8_t transcode_weights[BLOCK_W * BLOCK_H * 2]; requantize_astc_weights(total_grid_weights, log_blk.m_weights, bmd.m_weight_ise_range, transcode_weights, bmd.m_transcode_weight_ise_range); - copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk); + copy_weight_grid(bmd.m_dp, bmd.m_grid_x, bmd.m_grid_y, transcode_weights, decomp_blk, orig_behavior); status = astc_helpers::pack_astc_block(phys_blk, decomp_blk); if (!status) @@ -23971,4 +24746,17029 @@ namespace basist #endif // BASISD_SUPPORT_UASTC_HDR +#if BASISD_SUPPORT_XUASTC +namespace astc_ldr_t +{ + bool g_initialized; + astc_block_grid_data_hash_t g_astc_block_grid_data_hash; + + // Used for quickly bumping up or down quantized, 2 complement+shifted base+offset delta values without disturbing the MSB. + static basisu::vector g_base_ofs_nudges[astc_helpers::BISE_256_LEVELS + 1][2]; // [endpoint_ise_range][pos=0, neg=1] + + const int s_unique_ldr_index_to_astc_cem[6] = + { + astc_helpers::CEM_LDR_LUM_DIRECT, + astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE, + astc_helpers::CEM_LDR_RGB_DIRECT, + astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A, + astc_helpers::CEM_LDR_RGBA_DIRECT + }; + + static void compute_base_ofs_requantize_tabs() + { + for (uint32_t e_ise_range = astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; e_ise_range <= astc_helpers::LAST_VALID_ENDPOINT_ISE_RANGE; e_ise_range++) + { + const uint32_t num_levels = astc_helpers::get_ise_levels(e_ise_range); + + for (uint32_t pos_or_neg = 0; pos_or_neg < 2; pos_or_neg++) + { + g_base_ofs_nudges[e_ise_range][pos_or_neg].resize(num_levels); + + const int delta = pos_or_neg ? -1 : 1; + + for (uint32_t cur_ise = 0; cur_ise < num_levels; cur_ise++) + { + int cur_dequant = astc_helpers::g_dequant_tables.get_endpoint_tab(e_ise_range).m_ISE_to_val[cur_ise]; + + int cur_a = cur_dequant, cur_b = 0; + astc_helpers::bit_transfer_signed_dec(cur_a, cur_b); + + int best_err = INT_MAX; + uint32_t best_trial_ise = 0; + + for (uint32_t trial_ise = 0; trial_ise < num_levels; trial_ise++) + { + int trial_dequant = astc_helpers::g_dequant_tables.get_endpoint_tab(e_ise_range).m_ISE_to_val[trial_ise]; + + int trial_a = trial_dequant, trial_b = 0; + astc_helpers::bit_transfer_signed_dec(trial_a, trial_b); + + // ensure the transferred bit hasn't changed + if (cur_b != trial_b) + continue; + + // skip if the decoded delta hasn't changed at all + if (trial_a == cur_a) + continue; + + // do they want to nudge neg or pos + if (delta < 0) + { + // neg nudge, but trial delta is higher + if (trial_a > cur_a) + continue; + } + else + { + // pos nudge, but trial delta is lower + if (trial_a < cur_a) + continue; + } + + int e = basisu::iabs(trial_a - cur_a); + if (e < best_err) + { + best_err = e; + best_trial_ise = trial_ise; + } + } // trial_ise + + if (best_err == INT_MAX) + { + //fmt_printf("Failed nudge: eise:{}, delta: {}, curise:{}, cura:{}, curb:{}\n", e_ise_range, delta, cur_ise, cur_a, cur_b); + + // Failed to nudge, leave it unchanged + best_trial_ise = cur_ise; + } + + g_base_ofs_nudges[e_ise_range][pos_or_neg][cur_ise] = (uint8_t)best_trial_ise; + + } // cur_ise + + } // pos_or_neg + + } // e_ise_range + } + + void init() + { + if (g_initialized) + return; + + g_initialized = true; + + init_astc_block_grid_data_hash(); + + compute_base_ofs_requantize_tabs(); + } + + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b) + { + color_rgba enc; + + int tr = orig.r * 2 - encoded_b; + int tg = orig.g * 2 - encoded_b; + if ((tr < 0) || (tr > 255) || (tg < 0) || (tg > 255)) + did_clamp = true; + + enc.r = (uint8_t)basisu::clamp(tr, 0, 255); + enc.g = (uint8_t)basisu::clamp(tg, 0, 255); + enc.b = (uint8_t)orig.b; + enc.a = orig.a; + return enc; + } + + color_rgba blue_contract_dec(int enc_r, int enc_g, int enc_b, int enc_a) + { + color_rgba dec; + dec.r = (uint8_t)((enc_r + enc_b) >> 1); + dec.g = (uint8_t)((enc_g + enc_b) >> 1); + dec.b = (uint8_t)enc_b; + dec.a = (uint8_t)enc_a; + return dec; + } + + static inline int quant_preserve2(uint32_t ise_range, uint32_t v) + { + if (ise_range == astc_helpers::BISE_256_LEVELS) + return v; + + assert(ise_range >= astc_helpers::BISE_6_LEVELS); + + return basist::astc_6x6_hdr::g_quantize_tables_preserve2[ise_range][v]; + } + + //---------------------------------------------------------------------------------- + // Requantize endpoints, but preserves blue contraction and base+ofs bits as much as possible. + + // Blue contraction should be preserved almost always if quantizing down, except with base+ofs (extremely to incredibly rare). + // endpoints never swapped for base+ofs + // NOTE: Cannot use any floating point math for determinism across compilers. + bool requantize_ise_endpoints(uint32_t cem, + uint32_t src_ise_endpoint_range, const uint8_t* pSrc_endpoints, + uint32_t dst_ise_endpoint_range, uint8_t* pDst_endpoints) + { + if (!astc_helpers::is_cem_ldr(cem)) + { + assert(0); + return false; + } + + const uint32_t num_endpoint_vals = astc_helpers::get_num_cem_values(cem); + assert(num_endpoint_vals <= astc_helpers::MAX_CEM_ENDPOINT_VALS); + + if (src_ise_endpoint_range == dst_ise_endpoint_range) + { + memcpy(pDst_endpoints, pSrc_endpoints, num_endpoint_vals); + return true; + } + + uint8_t dequantized_src_vals_temp[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + const uint8_t* pDequantized_src_vals = pSrc_endpoints; + + if (src_ise_endpoint_range != astc_helpers::BISE_256_LEVELS) + { + const auto& dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + dequantized_src_vals_temp[i] = dequant_tab[pSrc_endpoints[i]]; + + pDequantized_src_vals = dequantized_src_vals_temp; + } + + if (dst_ise_endpoint_range == astc_helpers::BISE_256_LEVELS) + { + memcpy(pDst_endpoints, pDequantized_src_vals, num_endpoint_vals); + return true; + } + + const auto& dst_quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_val_to_ise; + + if ((cem == astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) || (cem == astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + // preserve v1,v3,v5,v7, which have 2 MSB's that need to be preserved during requant + if (i & 1) + pDst_endpoints[i] = (uint8_t)quant_preserve2(dst_ise_endpoint_range, pDequantized_src_vals[i]); + else + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + } + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + + if (i & 1) + { + assert((src_v & 64) == (dst_v & 64)); + } + } + } +#endif + + const bool src_used_blue_contract = astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range); + // src delta sum was < 0 if it used blue contraction, >= 0 if it did NOT + + int v0 = dst_dequant_tab[pDst_endpoints[0]], v1 = dst_dequant_tab[pDst_endpoints[1]]; + int v2 = dst_dequant_tab[pDst_endpoints[2]], v3 = dst_dequant_tab[pDst_endpoints[3]]; + int v4 = dst_dequant_tab[pDst_endpoints[4]], v5 = dst_dequant_tab[pDst_endpoints[5]]; + + astc_helpers::bit_transfer_signed_dec(v1, v0); + astc_helpers::bit_transfer_signed_dec(v3, v2); + astc_helpers::bit_transfer_signed_dec(v5, v4); + + int s = v1 + v3 + v5; + bool quant_used_blue_contraction = (s < 0); + + // Kind of a dumb algorithm, but it only tries 2-3 times in random testing. + //const uint32_t MAX_TRIES = 10; + const uint32_t MAX_TRIES = 5; + + uint32_t tries = 0; + + if (src_used_blue_contract != quant_used_blue_contraction) + { + int nudge_delta = quant_used_blue_contraction ? 1 : -1; + + uint32_t cur_c_rover = 2; // b first + + for (tries = 0; tries < MAX_TRIES; tries++) + { + for (uint32_t j = 0; j < 3; j++) + { + const uint32_t i = (cur_c_rover + j) % 3; + + // This will either nudge the delta, or fail because it's either at the [-32,31] limit or it can't go further in the desired delta direction due to quantization limits + uint32_t new_ise_v = g_base_ofs_nudges[dst_ise_endpoint_range][(nudge_delta < 0) ? 1 : 0][pDst_endpoints[1 + i * 2]]; + + if (new_ise_v != pDst_endpoints[1 + i * 2]) + { + // It changed, so a successful nudge, but the base MSB should be preserved + pDst_endpoints[1 + i * 2] = (uint8_t)new_ise_v; + break; + } + } + + v0 = dst_dequant_tab[pDst_endpoints[0]], v1 = dst_dequant_tab[pDst_endpoints[1]]; + v2 = dst_dequant_tab[pDst_endpoints[2]], v3 = dst_dequant_tab[pDst_endpoints[3]]; + v4 = dst_dequant_tab[pDst_endpoints[4]], v5 = dst_dequant_tab[pDst_endpoints[5]]; + + astc_helpers::bit_transfer_signed_dec(v1, v0); + astc_helpers::bit_transfer_signed_dec(v3, v2); + astc_helpers::bit_transfer_signed_dec(v5, v4); + + s = v1 + v3 + v5; + quant_used_blue_contraction = (s < 0); + + if (src_used_blue_contract == quant_used_blue_contraction) + break; + + ++cur_c_rover; + + } // tries + } + + if (tries < MAX_TRIES) + { + assert(astc_helpers::used_blue_contraction(cem, pDst_endpoints, dst_ise_endpoint_range) == astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range)); + } + else + { + // It failed to adjust, ultimately harmless as we have RGB(A) direct anyway (and at this likely very low quant level, it won't matter). + // TODO: We could try more adjustments, but this seems extremely unlikely to be worth the trouble after random testing. + +#if BASISU_ASTC_LDR_DEBUG_MSGS + static bool s_msg_printed = false; + if (!s_msg_printed) + fmt_debug_printf("requantize_ise_endpoints: blue contraction enforcement failed\n"); +#endif + } + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + } + } +#endif + } + else if ((cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + // See if the original colors were blue contracted + uint32_t s0 = pDequantized_src_vals[0] + pDequantized_src_vals[2] + pDequantized_src_vals[4]; + uint32_t s1 = pDequantized_src_vals[1] + pDequantized_src_vals[3] + pDequantized_src_vals[5]; + + const bool orig_used_blue_contract = s1 < s0; + + for (uint32_t i = 0; i < num_endpoint_vals; i++) + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + + uint32_t dequant_s0 = dst_dequant_tab[pDst_endpoints[0]] + dst_dequant_tab[pDst_endpoints[2]] + dst_dequant_tab[pDst_endpoints[4]]; + uint32_t dequant_s1 = dst_dequant_tab[pDst_endpoints[1]] + dst_dequant_tab[pDst_endpoints[3]] + dst_dequant_tab[pDst_endpoints[5]]; + + const bool quant_used_blue_contract = dequant_s1 < dequant_s0; + + if (orig_used_blue_contract != quant_used_blue_contract) + { + if (dequant_s0 == dequant_s1) + { + assert(orig_used_blue_contract); + assert(!quant_used_blue_contract); + + // swapping won't work because sums are equal, so force dst to use blue contraction by nudgling a component + // original s1=requant_s0 + + if (dequant_s1) + { + // decrease s1 + for (uint32_t i = 0; i < 3; i++) + { + uint32_t new_ise_v = astc_helpers::apply_delta_to_bise_endpoint_val(dst_ise_endpoint_range, pDst_endpoints[1 + i * 2], -1); + if (new_ise_v != pDst_endpoints[1 + i * 2]) + { + pDst_endpoints[1 + i * 2] = (uint8_t)new_ise_v; + break; + } + } + } + else + { + // both are 0, increase s0 + for (uint32_t i = 0; i < 3; i++) + { + uint32_t new_ise_val = astc_helpers::apply_delta_to_bise_endpoint_val(dst_ise_endpoint_range, pDst_endpoints[i * 2], 1); + if (new_ise_val != pDst_endpoints[i * 2]) + { + pDst_endpoints[i * 2] = (uint8_t)new_ise_val; + break; + } + } + } + } + else + { + std::swap(pDst_endpoints[0], pDst_endpoints[1]); + std::swap(pDst_endpoints[2], pDst_endpoints[3]); + std::swap(pDst_endpoints[4], pDst_endpoints[5]); + + if (cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + std::swap(pDst_endpoints[6], pDst_endpoints[7]); + } + } + + assert(astc_helpers::used_blue_contraction(cem, pDst_endpoints, dst_ise_endpoint_range) == astc_helpers::used_blue_contraction(cem, pSrc_endpoints, src_ise_endpoint_range)); + } + else + { + for (uint32_t i = 0; i < num_endpoint_vals; i++) + pDst_endpoints[i] = dst_quant_tab[pDequantized_src_vals[i]]; + +#ifdef _DEBUG + { + const auto& src_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(src_ise_endpoint_range).m_ISE_to_val; + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_ise_endpoint_range).m_ISE_to_val; + + // ensure MSB's did not change + for (uint32_t i = 0; i < num_endpoint_vals; i++) + { + int src_v = src_dequant_tab[pSrc_endpoints[i]]; + int dst_v = dst_dequant_tab[pDst_endpoints[i]]; + assert((src_v & 128) == (dst_v & 128)); + } + } +#endif + } + + return true; + } + + // First packs base+ofs to ise20 (always enforcing blue contraction), then quantizes down (preserving blue contraction whenever possible, which might possibly not be in extreme quantizations). + // NOTE: Cannot use any floating point math for determinism across compilers. + bool pack_base_offset( + uint32_t cem_index, uint32_t dst_ise_endpoint_range, uint8_t* pPacked_endpoints, + const color_rgba& l, const color_rgba& h, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag, bool& endpoints_swapped) + { + blue_contraction_clamped_flag = false; + base_ofs_clamped_flag = false; + endpoints_swapped = false; + + if ((cem_index != astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET) && (cem_index != astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET)) + { + assert(0); + return false; + } + + color_rgba pack_l(l), pack_h(h); + + if (use_blue_contraction) + { + color_rgba enc_l(blue_contract_enc(pack_l, blue_contraction_clamped_flag, pack_l.b)); + color_rgba enc_h(blue_contract_enc(pack_h, blue_contraction_clamped_flag, pack_h.b)); + + if ((blue_contraction_clamped_flag) && (auto_disable_blue_contraction_if_clamped)) + { + use_blue_contraction = false; + } + else + { + pack_h = enc_l; + pack_l = enc_h; + + endpoints_swapped = true; + } + } + + int dr = 0, dg = 0, db = 0, da = 0; + bool pack_uses_blue_contraction = false; + int low_clamp = -32; + + // first 2 passes try with swapping by asymmetric clamping, then next 2 try with symmetric clamping, should always succeed + for (uint32_t pass = 0; pass < 4; pass++) + { + // Take previous CEM's values and try to encode to base+offset as best we can, it may clamp + int orig_dr = pack_h.r - pack_l.r, orig_dg = pack_h.g - pack_l.g, orig_db = pack_h.b - pack_l.b, orig_da = pack_h.a - pack_l.a; + + base_ofs_clamped_flag = false; + + dr = basisu::clamp(orig_dr, low_clamp, 31); + if (dr != orig_dr) base_ofs_clamped_flag = true; + + dg = basisu::clamp(orig_dg, low_clamp, 31); + if (dg != orig_dg) base_ofs_clamped_flag = true; + + db = basisu::clamp(orig_db, low_clamp, 31); + if (db != orig_db) base_ofs_clamped_flag = true; + + da = basisu::clamp(orig_da, low_clamp, 31); + if (da != orig_da) base_ofs_clamped_flag = true; + + int s = dr + dg + db; + + pack_uses_blue_contraction = s < 0; + + if (pack_uses_blue_contraction == use_blue_contraction) + break; + + if (s == 0) + { + assert(!pack_uses_blue_contraction); + assert(use_blue_contraction); + + // !pack_uses_blue_contraction here, sum=0, so force sum negative + if (db > -32) + db--; + else if (dr > -32) + dr--; + else if (dg > -32) + dg--; + else + { + // they can't be all -32 (or negative), otherwise sum couldn't be 0 + assert(0); + } + + assert((dr + dg + db) < 0); + + pack_uses_blue_contraction = true; + + break; + } + + if (pass == 3) + { + // theoretically unreachable + assert(0); + break; + } + + if (pass == 1) + { + // Try 2 more swap passes, but enforce a symmetric clamp range - this *should* work. + low_clamp = -31; + } + + std::swap(pack_l, pack_h); + endpoints_swapped = !endpoints_swapped; + + } // pass + + int v0 = pack_l.r, v2 = pack_l.g, v4 = pack_l.b; + int v1 = dr, v3 = dg, v5 = db; + + // lossless at 8-bits + astc_helpers::bit_transfer_signed_enc(v1, v0); + astc_helpers::bit_transfer_signed_enc(v3, v2); + astc_helpers::bit_transfer_signed_enc(v5, v4); + + int v6 = 0, v7 = 0; + if (astc_helpers::does_cem_have_alpha(cem_index)) + { + v6 = pack_l.a; + v7 = da; + + astc_helpers::bit_transfer_signed_enc(v7, v6); + } + + uint8_t new_endpoints8[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + + new_endpoints8[0] = (uint8_t)v0; + new_endpoints8[1] = (uint8_t)v1; + + new_endpoints8[2] = (uint8_t)v2; + new_endpoints8[3] = (uint8_t)v3; + + new_endpoints8[4] = (uint8_t)v4; + new_endpoints8[5] = (uint8_t)v5; + + if (astc_helpers::does_cem_have_alpha(cem_index)) + { + new_endpoints8[6] = (uint8_t)v6; + new_endpoints8[7] = (uint8_t)v7; + } + + // This should always succeed. + assert(astc_helpers::used_blue_contraction(cem_index, new_endpoints8, astc_helpers::BISE_256_LEVELS) == use_blue_contraction); + + // requant predicted 256 level endpoints to current endpoint quant level, this will nearly always (if not always) succeed + bool status = requantize_ise_endpoints( + cem_index, astc_helpers::BISE_256_LEVELS, new_endpoints8, + dst_ise_endpoint_range, pPacked_endpoints); + + // can't assert because requant to a very low quant level could have failed to preserve blue contraction (in practice, super rare, perhaps impossible - still determining) + //assert(astc_helpers::used_blue_contraction(cem_index, pPredicted_endpoints, cur_blk.m_endpoint_ise_range) == pack_uses_blue_contraction); + + return status; + } + + // converts a previous block's endpoints, using any supported LDR CEM/endpoint quant level, into a new CEM/endpoint quant level + // used for prediction or *potentially* coding purposes + // will return num_dst_endpoint_vals residuals in cur_blk's endpoint level quant + // NOTE: Cannot use any floating point math for determinism across compilers. + bool convert_endpoints_across_cems( + uint32_t prev_cem, uint32_t prev_endpoint_ise_range, const uint8_t* pPrev_endpoints, + uint32_t dst_cem, uint32_t dst_endpoint_ise_range, uint8_t* pDst_endpoints, + bool always_repack, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag) + { + blue_contraction_clamped_flag = false; + base_ofs_clamped_flag = false; + + const uint32_t num_dst_endpoint_vals = astc_helpers::get_num_cem_values(dst_cem); + + const auto& dst_quant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_endpoint_ise_range).m_val_to_ise; + const auto& dst_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(dst_endpoint_ise_range).m_ISE_to_val; + + if ((prev_cem == dst_cem) && (!always_repack)) + { + // CEM's are precisely equal + // Requantize prev block's endpoints into the current block's quant levels, and we're done + return requantize_ise_endpoints( + prev_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + } + + // CEM's cannot be precisely equal now, compute base CEM's (removing alpha from consideration) + if (!always_repack) + { + // this path preserves the original's blue contraction status + const uint32_t prev_base_cem = astc_helpers::get_base_cem_without_alpha(prev_cem); + const uint32_t dst_base_cem = astc_helpers::get_base_cem_without_alpha(dst_cem); + + // prev cem has alpha, cur cem doesn't, but otherwise modes identical, so it's being stripped + if ((prev_base_cem == dst_base_cem) && (!astc_helpers::does_cem_have_alpha(dst_cem))) + { + assert(astc_helpers::does_cem_have_alpha(prev_cem)); + assert(astc_helpers::get_num_cem_values(prev_base_cem) == num_dst_endpoint_vals); + + // Requantize prev block's endpoints into the current block's quant levels, but ignore the alpha values (which are always the last 2 entries) + return requantize_ise_endpoints( + prev_base_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + } + + // if prev cem doesn't have alpha, but the current cem does, but otherwise modes are identical, so add sane alpha (both 255) and hope for best in the prediction + if ((prev_base_cem == dst_base_cem) && (astc_helpers::does_cem_have_alpha(dst_cem))) + { + assert(!astc_helpers::does_cem_have_alpha(prev_base_cem)); + + // requant previous endpoints to current endpoint quant level + bool status = requantize_ise_endpoints( + prev_base_cem, prev_endpoint_ise_range, pPrev_endpoints, + dst_endpoint_ise_range, pDst_endpoints); + + if (!status) + return false; + + // just plug in 255 to both alphas + const int ise_a_val = dst_quant_tab[255]; + + switch (dst_cem) + { + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + assert(num_dst_endpoint_vals == 4); + pDst_endpoints[2] = (uint8_t)ise_a_val; + pDst_endpoints[3] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + assert(num_dst_endpoint_vals == 8); + pDst_endpoints[6] = (uint8_t)ise_a_val; + pDst_endpoints[7] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: + { + assert(num_dst_endpoint_vals == 6); + pDst_endpoints[4] = (uint8_t)ise_a_val; + pDst_endpoints[5] = (uint8_t)ise_a_val; + break; + } + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + assert(num_dst_endpoint_vals == 8); + + // alphas should decode to 255, 255 + pDst_endpoints[6] = (uint8_t)ise_a_val; // base + pDst_endpoints[7] = (uint8_t)dst_quant_tab[128]; // offset, top bit should be preserved + break; + } + default: + assert(0); + break; + } + + return true; + } + + } // !always_repack + + // Here the CEM's are not even in the same class. + // Do something reasonable to convert to the current block's CEM encoding and hope the residual distribution is reasonable. + + // fully decode the endpoints, undoing any quant, blue contraction or bit transfers + color_rgba prev_l, prev_h; + decode_endpoints(prev_cem, pPrev_endpoints, prev_endpoint_ise_range, prev_l, prev_h); + + uint8_t new_endpoints8[astc_helpers::MAX_CEM_ENDPOINT_VALS] = { 0 }; + + // now pack the endpoints to the desired CEM + switch (dst_cem) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: + { + // Take previous endpoints and convert to luma/alpha low/high. + new_endpoints8[0] = (prev_l.r + prev_l.g + prev_l.b + 1) / 3; + new_endpoints8[1] = (prev_h.r + prev_h.g + prev_h.b + 1) / 3; + + if (dst_cem == astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT) + { + new_endpoints8[2] = prev_l.a; + new_endpoints8[3] = prev_h.a; + } + + // ensure L new_endpoints8[1]) + { + std::swap(new_endpoints8[0], new_endpoints8[1]); + std::swap(new_endpoints8[2], new_endpoints8[3]); + } + } + + // requant predicted 256 level endpoints to current endpoint quant level + return requantize_ise_endpoints(dst_cem, astc_helpers::BISE_256_LEVELS, new_endpoints8, dst_endpoint_ise_range, pDst_endpoints); + } + + case astc_helpers::CEM_LDR_RGB_DIRECT: + case astc_helpers::CEM_LDR_RGBA_DIRECT: + { + // Take previous endpoints and convert to rgb(a) direct, explictly preserving previous ordering (to preserve the previous's endpoints usage of blue contraction, if it used it). + new_endpoints8[0] = prev_l.r; + new_endpoints8[1] = prev_h.r; + + new_endpoints8[2] = prev_l.g; + new_endpoints8[3] = prev_h.g; + + new_endpoints8[4] = prev_l.b; + new_endpoints8[5] = prev_h.b; + + if (dst_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + { + new_endpoints8[6] = prev_l.a; + new_endpoints8[7] = prev_h.a; + } + + if (use_blue_contraction) + { + color_rgba enc_l(blue_contract_enc(prev_l, blue_contraction_clamped_flag, dst_dequant_tab[dst_quant_tab[prev_l.b]])); + color_rgba enc_h(blue_contract_enc(prev_h, blue_contraction_clamped_flag, dst_dequant_tab[dst_quant_tab[prev_h.b]])); + + if ((auto_disable_blue_contraction_if_clamped) && (blue_contraction_clamped_flag)) + { + use_blue_contraction = false; + } + else + { + new_endpoints8[0] = enc_h.r; + new_endpoints8[1] = enc_l.r; + + new_endpoints8[2] = enc_h.g; + new_endpoints8[3] = enc_l.g; + + new_endpoints8[4] = enc_h.b; + new_endpoints8[5] = enc_l.b; + + if (dst_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + { + new_endpoints8[6] = prev_h.a; + new_endpoints8[7] = prev_l.a; + } + } + } + + uint32_t s0 = new_endpoints8[0] + new_endpoints8[2] + new_endpoints8[4]; + uint32_t s1 = new_endpoints8[1] + new_endpoints8[3] + new_endpoints8[5]; + bool pack_used_blue_contraction = s1 < s0; + + if (pack_used_blue_contraction != use_blue_contraction) + { + if (s0 == s1) + { + assert(!pack_used_blue_contraction); + + // swapping won't work because sums are equal, so force dst to use blue contraction by nudgling a component + // require s1 (hc.r + hc.g + hc.b)) + { + std::swap(lc, hc); + } + } + + new_endpoints8[0] = hc.r; + new_endpoints8[1] = hc.g; + new_endpoints8[2] = hc.b; + +#if 0 + // TODO: remove FP here + vec3F lf((float)lc.r, (float)lc.g, (float)lc.b); + vec3F hf((float)hc.r, (float)hc.g, (float)hc.b); + + const float MAX_S = 255.0f / 256.0f; + + float scale = MAX_S; + + float d = lf.dot(hf); + float nrm = hf.norm(); + if (nrm > 0.0f) + scale = d / nrm; + + scale = basisu::clamp(scale, 0.0f, MAX_S); + + new_endpoints8[3] = (uint8_t)basisu::clamp((int)std::round(scale * 256.0f), 0, 255); // explictly not 255.0f, but 256.0f, decoder divides scale by 256.0f +#endif + + { + int id = (lc.r * hc.r) + (lc.g * hc.g) + (lc.b * hc.b); + int inrm = (hc.r * hc.r) + (hc.g * hc.g) + (hc.b * hc.b); + + const int IMAX_S = (1024 * 255) / 256; + + int iscale = IMAX_S; + if (inrm > 0) + iscale = (id * 1024) / inrm; + + iscale = basisu::clamp(iscale, 0, IMAX_S); + + iscale = (iscale + 2) >> 2; + iscale = basisu::clamp(iscale, 0, 255); + + //assert(basisu::iabs(new_endpoints8[3] - iscale) <= 1); + new_endpoints8[3] = static_cast(iscale); + } + + if (dst_cem == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A) + { + new_endpoints8[4] = lc.a; + new_endpoints8[5] = hc.a; + + if ((prev_cem != astc_helpers::CEM_LDR_RGB_BASE_SCALE) && + (prev_cem != astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A)) + { + // ensure val4 < val5 for proper lerping on single plane (correlated alpha) + if (new_endpoints8[4] > new_endpoints8[5]) + std::swap(new_endpoints8[4], new_endpoints8[5]); + } + } + + // requant predicted 256 level endpoints to current endpoint quant level + return requantize_ise_endpoints(dst_cem, astc_helpers::BISE_256_LEVELS, new_endpoints8, dst_endpoint_ise_range, pDst_endpoints); + } + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: + { + bool endpoints_swapped = false; + + return pack_base_offset(dst_cem, dst_endpoint_ise_range, pDst_endpoints, prev_l, prev_h, use_blue_contraction, auto_disable_blue_contraction_if_clamped, + blue_contraction_clamped_flag, base_ofs_clamped_flag, endpoints_swapped); + } + default: + { + assert(0); + return false; + } + } + + return true; + } + + // Assumes ise 20 (256 levels) + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color32& l, color32& h) + { + assert(astc_helpers::is_cem_ldr(cem_index)); + + int ldr_endpoints[4][2]; + astc_helpers::decode_endpoint(cem_index, ldr_endpoints, pEndpoint_vals); + + for (uint32_t c = 0; c < 4; c++) + { + assert((ldr_endpoints[c][0] >= 0) && (ldr_endpoints[c][0] <= 255)); + assert((ldr_endpoints[c][1] >= 0) && (ldr_endpoints[c][1] <= 255)); + + l[c] = (uint8_t)ldr_endpoints[c][0]; + h[c] = (uint8_t)ldr_endpoints[c][1]; + } + } + + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color32& l, color32& h, float* pScale) + { + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + for (uint32_t i = 0; i < total_endpoint_vals; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + decode_endpoints_ise20(cem_index, dequantized_endpoints, l, h); + + if ((pScale) && ((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A))) + { + *pScale = (float)dequantized_endpoints[3] * (1.0f / 256.0f); + } + } + + // Assumes ise 20 (256 levels) + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h) + { + assert(astc_helpers::is_cem_ldr(cem_index)); + + int ldr_endpoints[4][2]; + astc_helpers::decode_endpoint(cem_index, ldr_endpoints, pEndpoint_vals); + + for (uint32_t c = 0; c < 4; c++) + { + assert((ldr_endpoints[c][0] >= 0) && (ldr_endpoints[c][0] <= 255)); + assert((ldr_endpoints[c][1] >= 0) && (ldr_endpoints[c][1] <= 255)); + + l[c] = (uint8_t)ldr_endpoints[c][0]; + h[c] = (uint8_t)ldr_endpoints[c][1]; + } + } + + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale) + { + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(cem_index); + + const auto& endpoint_dequant_tab = astc_helpers::g_dequant_tables.get_endpoint_tab(endpoint_ise_index).m_ISE_to_val; + + uint8_t dequantized_endpoints[astc_helpers::MAX_CEM_ENDPOINT_VALS]; + for (uint32_t i = 0; i < total_endpoint_vals; i++) + dequantized_endpoints[i] = endpoint_dequant_tab[pEndpoint_vals[i]]; + + decode_endpoints_ise20(cem_index, dequantized_endpoints, l, h); + + if ((pScale) && ((cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE) || (cem_index == astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A))) + { + *pScale = (float)dequantized_endpoints[3] * (1.0f / 256.0f); + } + } + + // TODO: Duplicated in astc_hdr + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + astc_helpers::weighted_sample samples[astc_helpers::MAX_BLOCK_DIM * astc_helpers::MAX_BLOCK_DIM]; + basisu::clear_obj(samples); + + astc_helpers::compute_upsample_weights(block_width, block_height, grid_width, grid_height, samples); + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + upsample_matrix.resize_rows_cols(num_block_samples, num_grid_samples); + + basisu::vector weights(num_grid_samples); + + // compute which source sample(s) contribute to it. + for (uint32_t d = 0; d < num_block_samples; d++) + { + const astc_helpers::weighted_sample& ws = samples[d]; + + weights.set_all(0.0f); + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + float w = ws.m_weights[y][x] * (1.0f / 16.0f); + if (!w) + continue; + + assert((ws.m_src_x + x) < grid_width); + assert((ws.m_src_y + y) < grid_height); + + assert(weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] == 0.0f); + weights[(ws.m_src_x + x) + (ws.m_src_y + y) * grid_width] = w; + } // x + } // y + + for (uint32_t i = 0; i < num_grid_samples; i++) + upsample_matrix.at_row_col(d, i) = weights[i]; + + } // d + } + + // TODO: Only needed by ASTC LDR encoder + void compute_adjoint_downsample_matrix(basisu::vector& downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 2) && (block_width <= astc_helpers::MAX_BLOCK_DIM)); + assert((block_height >= 2) && (block_height <= astc_helpers::MAX_BLOCK_DIM)); + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + + const uint32_t num_block_samples = block_width * block_height; + const uint32_t num_grid_samples = grid_width * grid_height; + + // Compute upsample matrix: output num_block_samples (rows), input num_grid_samples (cols) + basisu::vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, block_width, block_height, grid_width, grid_height); + + basisu::vector Dinv(num_grid_samples); + for (uint32_t j = 0; j < num_grid_samples; j++) + { + float sum = 0.0f; + + for (uint32_t i = 0; i < num_block_samples; i++) + sum += upsample_matrix.at_row_col(i, j); + + if (sum > 0.0f) + Dinv[j] = 1.0f / sum; + } + + // Create downsample matrix: num_grid_samples rows, num_block_samples cols + downsample_matrix.resize(num_grid_samples * num_block_samples); + downsample_matrix.set_all(0.0f); + + for (uint32_t j = 0; j < num_grid_samples; ++j) + for (uint32_t i = 0; i < num_block_samples; ++i) + downsample_matrix[j * num_block_samples + i] = Dinv[j] * upsample_matrix.at_row_col(i, j); + } + + const astc_block_grid_data* find_astc_block_grid_data(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + auto find_res(g_astc_block_grid_data_hash.find(astc_block_grid_config(block_width, block_height, grid_width, grid_height))); + assert(find_res != g_astc_block_grid_data_hash.end()); + + return &find_res->second; + } + + void init_astc_block_grid_data_hash() + { + if (g_astc_block_grid_data_hash.size()) + return; + + g_astc_block_grid_data_hash.clear(); + g_astc_block_grid_data_hash.reserve(384); + + // TODO: Iterate over all valid block sizes more efficiently + for (uint32_t block_h = 4; block_h <= 12; block_h++) + { + for (uint32_t block_w = 4; block_w <= 12; block_w++) + { + if (!astc_helpers::is_valid_block_size(block_w, block_h)) + continue; + + for (uint32_t grid_h = 2; grid_h <= block_h; grid_h++) + { + for (uint32_t grid_w = 2; grid_w <= block_w; grid_w++) + { + const int bw = block_w, bh = block_h; + const int gw = grid_w, gh = grid_h; + + const int num_texels = bw * bh; + const int num_weights = gw * gh; + + basisu::vector2D upsample_matrix; + compute_upsample_matrix(upsample_matrix, bw, bh, gw, gh); + + float accum = 0.0f; + for (int t = 0; t < num_texels; ++t) + { + float row_sum_sq = 0.0f; + const float* row = &(upsample_matrix.get_ptr())[t * num_weights]; + + for (int i = 0; i < num_weights; ++i) + { + float w = row[i]; + row_sum_sq += w * w; + } + + accum += row_sum_sq; + } + + // estimate of MSE weight quantization reduction due to bilinear weight grid upsampling + // TODO: Gamma is used during encoding now, not transcoding. + const float weight_gamma = accum / (float)num_texels; + + astc_block_grid_data grid_data(weight_gamma); + grid_data.m_upsample_matrix = upsample_matrix; + + basisu::vector& downsample_matrix = grid_data.m_downsample_matrix; + compute_adjoint_downsample_matrix(downsample_matrix, bw, bh, gw, gh); + + auto res = g_astc_block_grid_data_hash.insert(astc_block_grid_config(bw, bh, gw, gh), grid_data); + assert(res.second); + BASISU_NOTE_UNUSED(res); + + } // grid_w + } // grid_h + + } // block_h + + } // block_w + } + +#include "basisu_idct.h" + +#if 0 + typedef void (*idct_1d_func_ptr)(const float* src, int src_stride, float* dst, int dst_stride); + + const idct_1d_func_ptr g_idct_1d_func_ptrs[11] = + { + idct_1d_2, + idct_1d_3, + idct_1d_4, + + idct_1d_5, + idct_1d_6, + idct_1d_7, + idct_1d_8, + + idct_1d_9, + idct_1d_10, + idct_1d_11, + idct_1d_12, + }; +#endif + + static inline void idct_2d(const float* pSrc, float* pDst, uint32_t num_rows, uint32_t num_cols) + { + assert((num_rows >= 2) && (num_rows <= 12)); + assert((num_cols >= 2) && (num_cols <= 12)); + + float temp[12 * 12]; + + // IDCT cols from src to temp + +#if 0 + // This works but uses slow WASM indirect calls + + const idct_1d_func_ptr pCol_xform = g_idct_1d_func_ptrs[num_rows - 2]; + for (uint32_t c = 0; c < num_cols; c++) + { + (*pCol_xform)(pSrc + c, num_cols, temp + c, num_cols); + } +#else + switch (num_rows) + { + case 2: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_2(pSrc + c, num_cols, temp + c, num_cols); + break; + case 3: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_3(pSrc + c, num_cols, temp + c, num_cols); + break; + case 4: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_4(pSrc + c, num_cols, temp + c, num_cols); + break; + case 5: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_5(pSrc + c, num_cols, temp + c, num_cols); + break; + case 6: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_6(pSrc + c, num_cols, temp + c, num_cols); + break; + case 7: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_7(pSrc + c, num_cols, temp + c, num_cols); + break; + case 8: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_8(pSrc + c, num_cols, temp + c, num_cols); + break; + case 9: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_9(pSrc + c, num_cols, temp + c, num_cols); + break; + case 10: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_10(pSrc + c, num_cols, temp + c, num_cols); + break; + case 11: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_11(pSrc + c, num_cols, temp + c, num_cols); + break; + case 12: + default: + for (uint32_t c = 0; c < num_cols; c++) + idct_1d_12(pSrc + c, num_cols, temp + c, num_cols); + break; + } +#endif + + // IDCT rows from temp to dst + +#if 0 + // This works but uses slow WASM indirect calls + const idct_1d_func_ptr pRow_xform = g_idct_1d_func_ptrs[num_cols - 2]; + + float* pTemp_row = temp; + float* pDst_row = pDst; + for (uint32_t r = 0; r < num_rows; r++) + { + (*pRow_xform)(pTemp_row, 1, pDst_row, 1); + pTemp_row += num_cols; + pDst_row += num_cols; + } +#else + float* pTemp_row = temp; + float* pDst_row = pDst; + switch (num_cols) + { + case 2: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_2(pTemp_row, 1, pDst_row, 1); + break; + case 3: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_3(pTemp_row, 1, pDst_row, 1); + break; + case 4: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_4(pTemp_row, 1, pDst_row, 1); + break; + case 5: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_5(pTemp_row, 1, pDst_row, 1); + break; + case 6: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_6(pTemp_row, 1, pDst_row, 1); + break; + case 7: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_7(pTemp_row, 1, pDst_row, 1); + break; + case 8: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_8(pTemp_row, 1, pDst_row, 1); + break; + case 9: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_9(pTemp_row, 1, pDst_row, 1); + break; + case 10: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_10(pTemp_row, 1, pDst_row, 1); + break; + case 11: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_11(pTemp_row, 1, pDst_row, 1); + break; + case 12: + default: + for (uint32_t r = 0; r < num_rows; r++, pTemp_row += num_cols, pDst_row += num_cols) + idct_1d_12(pTemp_row, 1, pDst_row, 1); + break; + } +#endif + } + + bool dct2f::init(uint32_t rows, uint32_t cols) + { + if ((rows < 2u) || (rows > cMaxSize) || + (cols < 2u) || (cols > cMaxSize)) + { + assert(0); + return false; + } + + m_rows = rows; + m_cols = cols; + + m_c_col.assign(m_rows * m_rows, 0.0f); + m_c_row.assign(m_cols * m_cols, 0.0f); + m_a_col.assign(m_rows, 0.0f); + m_a_row.assign(m_cols, 0.0f); + + const float pi = 3.14159265358979323846f; + + // alpha scaling + const float inv_m = 1.0f / static_cast(m_rows); + m_a_col[0] = sqrtf(inv_m); + for (uint32_t u = 1; u < m_rows; ++u) + m_a_col[u] = sqrtf(2.0f * inv_m); + + const float inv_n = 1.0f / static_cast(m_cols); + m_a_row[0] = sqrtf(inv_n); + for (uint32_t v = 1; v < m_cols; ++v) + m_a_row[v] = sqrtf(2.0f * inv_n); + + // cos tables + for (uint32_t u = 0; u < m_rows; ++u) + { + for (uint32_t x = 0; x < m_rows; ++x) + { + float angle = (pi * static_cast((2 * x + 1) * u)) / (2.0f * static_cast(m_rows)); + m_c_col[u * m_rows + x] = cosf(angle); + } + } + + for (uint32_t v = 0; v < m_cols; ++v) + { + for (uint32_t y = 0; y < m_cols; ++y) + { + float angle = (pi * static_cast((2 * y + 1) * v)) / (2.0f * static_cast(m_cols)); + m_c_row[v * m_cols + y] = cosf(angle); + } + } + + return true; + } + + void dct2f::forward(const float* pSrc, float* pDst, fvec& work) const + { + forward(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::inverse(const float* pSrc, float* pDst, fvec& work) const + { + inverse(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::inverse_check(const float* pSrc, float* pDst, fvec& work) const + { + inverse_check(pSrc, m_cols, pDst, m_cols, work); + } + + void dct2f::forward(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + + float* pWork = &work[0]; + + // horizontal + for (uint32_t x = 0; x < m; ++x) + { + const float* pRowIn = pSrc + x * src_stride; + float* pRowT = pWork + x * n; + for (uint32_t v = 0; v < n; ++v) + { + const float* pCv = &m_c_row[v * n]; + float s = 0.0f; + for (uint32_t y = 0; y < n; ++y) + { + s += pRowIn[y] * pCv[y]; + } + pRowT[v] = s * m_a_row[v]; + } + } + + // vertical + for (uint32_t v = 0; v < n; ++v) + { + for (uint32_t u = 0; u < m; ++u) + { + const float* pCu = &m_c_col[u * m]; + float s = 0.0f; + for (uint32_t x = 0; x < m; ++x) + { + s += pWork[x * n + v] * pCu[x]; + } + pDst[u * dst_stride + v] = s * m_a_col[u]; + } + } + } + + // src_stride/dst_stride must be m_cols + void dct2f::inverse(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + BASISU_NOTE_UNUSED(src_stride); + BASISU_NOTE_UNUSED(dst_stride); + +#if 0 + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + float* pWork = &work[0]; + + // vertical + for (uint32_t v = 0; v < n; ++v) // cols + { + float sums[cMaxSize] = { 0 }; + + for (uint32_t u = 0; u < m; ++u) // rows + { + if (((const uint32_t*)pSrc)[u * src_stride + v] == 0) + continue; + + float yU = pSrc[u * src_stride + v]; // most coeffs will be 0 + //if (yU == 0.0f) + // continue; + + yU *= m_a_col[u]; + + for (uint32_t x = 0; x < m; ++x) + { + const float cU = m_c_col[u * m + x]; + sums[x] += yU * cU; + } // x + + } // u + + for (uint32_t x = 0; x < m; ++x) + pWork[x * n + v] = sums[x]; + + } // v + + // horizontal + for (uint32_t x = 0; x < m; ++x) // rows + { + const float* pRowT = pWork + x * n; + float* pRowOut = pDst + x * dst_stride; + + for (uint32_t y = 0; y < n; ++y) // cols + { + float s = 0.0f; + for (uint32_t v = 0; v < n; ++v) // cols + { + const float cV = m_c_row[v * n + y]; + s += (pRowT[v] * m_a_row[v]) * cV; + } + pRowOut[y] = s; + } + } +#else + BASISU_NOTE_UNUSED(work); + assert(src_stride == m_cols); + assert(dst_stride == m_cols); + idct_2d(pSrc, pDst, m_rows, m_cols); +#endif + } + + void dct2f::inverse_check(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const + { + assert(m_rows && m_cols); + work.resize(m_rows * m_cols); + + const uint32_t m = m_rows, n = m_cols; + float* pWork = &work[0]; + + // vertical + for (uint32_t v = 0; v < n; ++v) + { + for (uint32_t x = 0; x < m; ++x) + { + float s = 0.0f; + for (uint32_t u = 0; u < m; ++u) + { + const float yU = pSrc[u * src_stride + v]; + const float cU = m_c_col[u * m + x]; + s += (yU * m_a_col[u]) * cU; + } + pWork[x * n + v] = s; + } + } + + // horizontal + for (uint32_t x = 0; x < m; ++x) // rows + { + const float* pRowT = pWork + x * n; + float* pRowOut = pDst + x * dst_stride; + + for (uint32_t y = 0; y < n; ++y) // cols + { + float s = 0.0f; + for (uint32_t v = 0; v < n; ++v) // cols + { + const float cV = m_c_row[v * n + y]; + s += (pRowT[v] * m_a_row[v]) * cV; + } + pRowOut[y] = s; + } + } + } + + static int* generate_zigzag_order(int width, int height) + { + assert((width > 0) && (height > 0)); + + const int total = width * height; + int* pOrder = (int*)malloc(total * sizeof(int)); + if (!pOrder) + return nullptr; + + int idx = 0; + for (int s = 0; s < (width + height - 1); ++s) + { + // Start x at max(0, s - height + 1), end at min(s, width - 1) + const int x_start = (s < height) ? 0 : (s - height + 1); + const int x_end = (s < width) ? s : (width - 1); + + // Diagonal size + const int diag_size = x_end - x_start + 1; + int* pDiag = (int*)malloc(diag_size * sizeof(int)); + if (!pDiag) + { + free(pOrder); + return nullptr; + } + + int j = 0; + for (int x = x_start; x <= x_end; ++x) + { + int y = s - x; + assert(j < diag_size); + pDiag[j++] = x + y * width; + } + + // Reverse if s is odd (alternate direction) + if ((s & 1) == 1) + { + for (int k = diag_size - 1; k >= 0; --k) + { + assert(idx < total); + pOrder[idx++] = pDiag[k]; + } + } + else + { + for (int k = 0; k < diag_size; ++k) + { + assert(idx < total); + pOrder[idx++] = pDiag[k]; + } + } + + free(pDiag); + } + + return pOrder; + } + + static const int g_baseline_jpeg_y[8][8] = + { + // DC element modified so bilinear fetches near (0,0) grab a smaller quant table value, protecting most important LF coefficients + { 4, 11, 10, 16, 24, 40, 51, 61 }, + { 12, 12, 14, 19, 26, 58, 60, 55 }, + { 14, 13, 16, 24, 40, 57, 69, 56 }, + { 14, 17, 22, 29, 51, 87, 80, 62 }, + { 18, 22, 37, 56, 68,109,103, 77 }, + { 24, 35, 55, 64, 81,104,113, 92 }, + { 49, 64, 78, 87,103,121,120,101 }, + { 72, 92, 95, 98,112,100,103, 99 } + }; + + // centers at (0,0) + static inline float sample_jpeg_quant(const int Q8[8][8], float i, float j) + { + i = basisu::minimum(basisu::maximum(i, 0.0f), 7.0f); + j = basisu::minimum(basisu::maximum(j, 0.0f), 7.0f); + int i0 = (int)floorf(i), j0 = (int)floorf(j); + int i1 = basisu::minimum(i0 + 1, 7), j1 = basisu::minimum(j0 + 1, 7); + float ti = i - i0, tj = j - j0; + float a = (1 - ti) * Q8[j0][i0] + ti * Q8[j0][i1]; + float b = (1 - ti) * Q8[j1][i0] + ti * Q8[j1][i1]; + return (1 - tj) * a + tj * b; + } + + void grid_weight_dct::init(uint32_t block_width, uint32_t block_height) + { + m_block_width = block_width; + m_block_height = block_height; + + for (uint32_t grid_height = 2; grid_height <= block_height; grid_height++) + { + for (uint32_t grid_width = 2; grid_width <= block_width; grid_width++) + { + // Check if this is a valid ASTC weight grid dimension + if ((grid_width * grid_height) > astc_helpers::MAX_GRID_WEIGHTS) + continue; + + auto ins_res = m_grid_dim_key_vals.insert(grid_dim_key(grid_width, grid_height), grid_dim_value()); + auto& val = ins_res.first->second; + + val.m_dct.init(grid_height, grid_width); + + int* pZigZag = generate_zigzag_order(grid_width, grid_height); + + basisu::int_vec v(grid_width * grid_height); + memcpy(v.data(), pZigZag, sizeof(int) * grid_width * grid_height); + + free(pZigZag); + + val.m_zigzag.swap(v); + + } // w + } // h + } + + // This can used FP as it only impacts the final decoded weights (not future blocks) + bool grid_weight_dct::decode_block_weights( + float q, uint32_t plane_index, // plane of weights to decode and IDCT from stream + astc_helpers::log_astc_block& log_blk, // must be initialized except for the plane weights which are decoded + basist::bitwise_decoder* pDec, + const astc_block_grid_data* pGrid_data, // grid data for this grid size + block_stats* pS, + fvec& dct_work, + const dct_syms* pSyms) const + { + const uint32_t grid_width = log_blk.m_grid_width, grid_height = log_blk.m_grid_height; + const uint32_t total_grid_samples = grid_width * grid_height; + const uint32_t num_planes = log_blk.m_dual_plane ? 2 : 1; + + //const auto& dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_ISE_to_val; + const auto& quant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_val_to_ise; + + auto grid_dim_vals_iter = m_grid_dim_key_vals.find(grid_dim_key(grid_width, grid_height)); + + if (grid_dim_vals_iter == m_grid_dim_key_vals.end()) + { + // Invalid grid dimension for this block size + assert(0); + return false; + } + + auto& grid_dim_vals = grid_dim_vals_iter->second; + + const float span_len = get_max_span_len(log_blk, plane_index); + + const float level_scale = compute_level_scale(q, span_len, pGrid_data->m_weight_gamma, grid_width, grid_height, log_blk.m_weight_ise_range); + + float scaled_weight_coding_scale = SCALED_WEIGHT_BASE_CODING_SCALE; + if (log_blk.m_weight_ise_range <= astc_helpers::BISE_8_LEVELS) + scaled_weight_coding_scale = 1.0f / 8.0f; + + float mean_weight = 0; + + if (pDec) + mean_weight = (float)pDec->decode_truncated_binary((uint32_t)(64.0f * scaled_weight_coding_scale) + 1) / (float)scaled_weight_coding_scale; + else if (pSyms) + mean_weight = (float)pSyms->m_dc_sym / (float)scaled_weight_coding_scale; + else + { + assert(0); + return false; + } + + if (pS) + { + pS->m_mean_weight = mean_weight; + pS->m_total_coded_acs = 0; + pS->m_max_ac_coeff = 0; + } + + float dct_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + const auto& zigzag = grid_dim_vals.m_zigzag; + + basisu::clear_obj(dct_weights); + + sample_quant_table_state quant_state; + quant_state.init(q, m_block_width, m_block_height, level_scale); + + if (pDec) + { + for (uint32_t zig_idx = 1; zig_idx < total_grid_samples; zig_idx++) + { + uint32_t run_len = pDec->decode_rice(m_zero_run); + + if ((run_len + zig_idx) > total_grid_samples) + return false; + + zig_idx += run_len; + + if (zig_idx >= total_grid_samples) + break; + + int sign = pDec->get_bits(1); + + int coeff = pDec->decode_rice(m_coeff); + + if (sign) + coeff = -coeff; + + int dct_idx = zigzag[zig_idx]; + + const uint32_t y = (uint32_t)dct_idx / grid_width; + const uint32_t x = (uint32_t)dct_idx % grid_width; + + //const int quant = dct_quant_tab[dct_idx]; + const int quant = sample_quant_table(quant_state, x, y); + //assert(quant == sample_quant_table(quant_state, x, y)); + + dct_weights[dct_idx] = dequant_deadzone(coeff, quant, DEADZONE_ALPHA, x, y); + + if (pS) + { + ++pS->m_total_coded_acs; + pS->m_max_ac_coeff = basisu::maximum(pS->m_max_ac_coeff, basisu::iabs(coeff)); + } + } + } + else + { + uint32_t zig_idx = 1; + uint32_t coeff_ofs = 0; + while (coeff_ofs < pSyms->m_coeffs.size()) + { + const uint32_t run_len = pSyms->m_coeffs[coeff_ofs].m_num_zeros; + const int coeff = pSyms->m_coeffs[coeff_ofs].m_coeff; + coeff_ofs++; + + if ((run_len + zig_idx) > total_grid_samples) + return false; + + zig_idx += run_len; + + if (zig_idx >= total_grid_samples) + break; + + assert(coeff != INT_MAX); + + int dct_idx = zigzag[zig_idx]; + + const uint32_t y = (uint32_t)dct_idx / grid_width; + const uint32_t x = (uint32_t)dct_idx % grid_width; + + //const int quant = dct_quant_tab[dct_idx]; + const int quant = sample_quant_table(quant_state, x, y); + //assert(quant == sample_quant_table(quant_state, x, y)); + + dct_weights[dct_idx] = dequant_deadzone(coeff, quant, DEADZONE_ALPHA, x, y); + + if (pS) + { + ++pS->m_total_coded_acs; + pS->m_max_ac_coeff = basisu::maximum(pS->m_max_ac_coeff, basisu::iabs(coeff)); + } + + zig_idx++; + } + } + + float idct_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + grid_dim_vals.m_dct.inverse(dct_weights, idct_weights, dct_work); + +#if defined(_DEBUG) || defined(DEBUG) + // Sanity check IDCT vs. less optimized variant + // Also quant table sanity check vs. sample_quant_table(). + { + float idct_weights_temp[astc_helpers::MAX_BLOCK_PIXELS]; + grid_dim_vals.m_dct.inverse_check(dct_weights, idct_weights_temp, dct_work); + + int dct_quant_tab[astc_helpers::MAX_BLOCK_PIXELS]; + compute_quant_table(q, grid_width, grid_height, level_scale, dct_quant_tab); + + for (uint32_t i = 0; i < grid_width * grid_height; i++) + { + assert(basisu::equal_tol(idct_weights[i], idct_weights_temp[i], .00125f)); + + assert(!i || (dct_quant_tab[i] == sample_quant_table(quant_state, i % grid_width, i / grid_width))); + } + } +#endif + + // Compute final grid weights + for (uint32_t y = 0; y < grid_height; y++) + for (uint32_t x = 0; x < grid_width; x++) + log_blk.m_weights[(x + y * grid_width) * num_planes + plane_index] = quant_tab[basisu::clamp(fast_roundf_int(mean_weight + idct_weights[x + y * grid_width]), 0, 64)]; + + return true; + } + + // results of calling scale_quant_steps() for each # of ASTC weight levels + static const float g_scale_quant_steps[12] = { 1.51333141f, 1.41198814f, 1.35588217f, 1.31743157f, 1.28835952f, 1.24573100f, 1.21481407f, 1.19067919f, 1.15431654f, 1.12734985f, 1.10601568f, 1.07348967f }; + + // Adaptive quantization + float grid_weight_dct::compute_level_scale(float q, float span_len, float weight_gamma, uint32_t grid_width, uint32_t grid_height, uint32_t weight_ise_range) const + { + BASISU_NOTE_UNUSED(weight_gamma); + BASISU_NOTE_UNUSED(grid_width); + BASISU_NOTE_UNUSED(grid_height); + + assert((weight_ise_range >= astc_helpers::BISE_2_LEVELS) && (weight_ise_range <= astc_helpers::BISE_32_LEVELS)); + + // Standard JPEG quality factor calcs + // TODO: Precompute this once + float level_scale; + q = basisu::clamp(q, 1.0f, 100.0f); + if (q < 50.0f) + level_scale = 5000.0f / q; + else + level_scale = 200.0f - 2.0f * q; + + level_scale *= (1.0f / 100.0f); // because JPEG's quant table is scaled by 100 + + //const float span_floor = 28.0f; + const float span_floor = 14.0f; + //const float adaptive_factor = 255.0f / maximum(span_len, span_floor); + // 64.0 = dynamic range adjustment (JPEG uses 255) + // divide by span len to adjustment adaptive low/high values per-block (JPEG always uses effective span=0-255) + // actually (64/255) * 255/max(span_len, span_floor) + float adaptive_factor = 64.0f / basisu::maximum(span_len, span_floor); + + // input signal scalar quantization noise will be distributed between multiple AC coefficients - compensate by adaptively adjusting the quant step size + float weight_quant_adaptive_factor = g_scale_quant_steps[weight_ise_range]; + adaptive_factor *= weight_quant_adaptive_factor; + + // sanity + assert(fabs(weight_quant_adaptive_factor - scale_quant_steps(astc_helpers::get_ise_levels(weight_ise_range))) < .000125f); + + // Adjust for ASTC weight grid bilinear upsampling using precomputed constants depending on the weight grid dims (usually .5-1.0, smaller grids=lower weights) + // This compensates for weight quant error being smoothed out due to bilinear. + // It's unclear if this is actually useful, and looks worse on smaller weight grids. + //level_scale *= adaptive_factor / sqrtf(weight_gamma); // weight_gamma is power domain, not amplitude + + // (Adaptive quant) + level_scale *= adaptive_factor; + + // The higher the level_scale, the more quantized DCT coefficients will be and vice versa. + + return level_scale; + } + + int grid_weight_dct::sample_quant_table(sample_quant_table_state& state, uint32_t x, uint32_t y) const + { + assert(x || y); + + if (state.m_q >= 100.0f) + return 1; + + float ny = float(y); + float ry = ny * state.m_sy; + + float nx = float(x); + float rx = nx * state.m_sx; + + assert(x || y); + + // sample from the JPEG baseline luma 8x8 DCT quant matrix + // this is an approximation (we could do an actual desired radians per spatial sample search vs. each of the 8x8 basis vectors to find the best, most conservative mapping), + // but for 4x4 and 6x6 block sizes it's reasonable enough and simple/fast + // at 4x4, the lowest frequencies are slightly more heavily quantized than we would want (but the quant table entries near DC are so similar it's doubtful it matters much if at all) + //float base = sample_jpeg_quant(g_baseline_jpeg_y, rx, ry); + + float base; + { + float i = rx, j = ry; + assert((i >= 0.0f) && (j >= 0.0f)); + + i = basisu::minimum(i, 7.0f); + j = basisu::minimum(j, 7.0f); + + int i0 = (int)(i), j0 = (int)(j); + int i1 = basisu::minimum(i0 + 1, 7), j1 = basisu::minimum(j0 + 1, 7); + + float ti = i - i0, tj = j - j0; + float a = (1 - ti) * g_baseline_jpeg_y[j0][i0] + ti * g_baseline_jpeg_y[j0][i1]; + float b = (1 - ti) * g_baseline_jpeg_y[j1][i0] + ti * g_baseline_jpeg_y[j1][i1]; + + base = (1 - tj) * a + tj * b; + } + + int quant_scale = (int)(base * state.m_level_scale + 0.5f); + + quant_scale = basisu::maximum(1, quant_scale); + + return quant_scale; + } + + void grid_weight_dct::compute_quant_table(float q, + uint32_t grid_width, uint32_t grid_height, + float level_scale, int* dct_quant_tab) const + { + assert(q > 0.0f); + + dct_quant_tab[0] = 1; + + if (q >= 100.0f) + { + for (uint32_t y = 0; y < grid_height; y++) + for (uint32_t x = 0; x < grid_width; x++) + if (x || y) + dct_quant_tab[x + y * grid_width] = 1; + return; + } + + const int Bx = m_block_width, By = m_block_height; + + const float sx = (float)8.0f / (float)Bx; + const float sy = (float)8.0f / (float)By; + + for (uint32_t y = 0; y < grid_height; y++) + { + float ny = float(y); + float ry = ny * sy; + + for (uint32_t x = y ? 0 : 1; x < grid_width; x++) + { + int quant_scale = 0; + + assert(x || y); + + float nx = float(x); + float rx = nx * sx; + + // sample from the JPEG baseline luma 8x8 DCT quant matrix + // this is an approximation (we could do an actual desired radians per spatial sample search vs. each of the 8x8 basis vectors to find the best, most conservative mapping), + // but for 4x4 and 6x6 block sizes it's reasonable enough and simple/fast + // at 4x4, the lowest frequencies are slightly more heavily quantized than we would want (but the quant table entries near DC are so similar it's doubtful it matters much if at all) + float base = sample_jpeg_quant(g_baseline_jpeg_y, rx, ry); + + //quant_scale = (int)std::floor(base * level_scale + 0.5f); + quant_scale = (int)(base * level_scale + 0.5f); + assert(quant_scale == (int)std::floor(base * level_scale + 0.5f)); + + quant_scale = basisu::maximum(1, quant_scale); + + dct_quant_tab[x + y * grid_width] = quant_scale; + } // x + } // y + } + + // Needed by AQ + float grid_weight_dct::get_max_span_len(const astc_helpers::log_astc_block& log_blk, uint32_t plane_index) const + { + float span_len = 0.0f; + + if (log_blk.m_dual_plane) + { + color32 l, h; + decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + for (uint32_t c = 0; c < 4; c++) + { + if (plane_index == 1) + { + if (c == log_blk.m_color_component_selector) + { + span_len += basisu::squaref((float)h[c] - (float)l[c]); + } + } + else + { + if (c != log_blk.m_color_component_selector) + { + span_len += basisu::squaref((float)h[c] - (float)l[c]); + } + } + } + + span_len = sqrtf(span_len); + } + else + { + for (uint32_t i = 0; i < log_blk.m_num_partitions; i++) + { + color32 l, h; + decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints + astc_helpers::get_num_cem_values(log_blk.m_color_endpoint_modes[0]) * i, log_blk.m_endpoint_ise_range, l, h); + + float part_span_len = sqrtf( + basisu::squaref((float)h.r - (float)l.r) + basisu::squaref((float)h.g - (float)l.g) + basisu::squaref((float)h.b - (float)l.b) + basisu::squaref((float)h.a - (float)l.a) + ); + + span_len = basisu::maximum(part_span_len, span_len); + } + } + + return span_len; + } + +#include "basisu_astc_cfgs.inl" + + void create_encoder_trial_modes_table(uint32_t block_width, uint32_t block_height, + basisu::vector& encoder_trial_modes, grouped_trial_modes& grouped_encoder_trial_modes, + bool print_debug_info, bool print_modes) + { + //interval_timer itm; + //itm.start(); + + uint32_t mode_index = 0; + uint32_t max_grid_width = 0, max_grid_height = 0, max_grid_samples = 0; + + //encoder_trial_modes.reserve(BU_TOTAL_ASTC_CFGS); + encoder_trial_modes.reserve(3072); + encoder_trial_modes.resize(0); + + grouped_encoder_trial_modes.clear(); + + for (uint32_t cfg_index = 0; cfg_index < BU_TOTAL_ASTC_CFGS; cfg_index++) + { + assert((cfg_index * 3 + 2) < std::size(s_astc_cfg_table)); + uint32_t packed_mode = s_astc_cfg_table[cfg_index * 3] | (s_astc_cfg_table[cfg_index * 3 + 1] << 8) | (s_astc_cfg_table[cfg_index * 3 + 2] << 16); + + uint32_t endpoint_ise_range, weight_ise_range, ccs_index, num_subsets, unique_cem_index, grid_wh; + +#define BU_UNPACK_FIELD(val, bits) do { val = packed_mode & ((1u << (bits)) - 1u); packed_mode >>= (bits); } while(0) + BU_UNPACK_FIELD(endpoint_ise_range, CFG_PACK_EISE_BITS); + BU_UNPACK_FIELD(weight_ise_range, CFG_PACK_WISE_BITS); + BU_UNPACK_FIELD(ccs_index, CFG_PACK_CCS_BITS); + BU_UNPACK_FIELD(num_subsets, CFG_PACK_SUBSETS_BITS); + BU_UNPACK_FIELD(unique_cem_index, CFG_PACK_CEM_BITS); + BU_UNPACK_FIELD(grid_wh, CFG_PACK_GRID_BITS); +#undef BU_UNPACK_FIELD + + assert(!packed_mode); + + const uint32_t grid_width = (grid_wh / 11) + 2; + + // modes are sorted by grid widths, which is at/near the MSB of the packed values, rest must be >= + if (grid_width > block_width) + break; + + const uint32_t grid_height = (grid_wh % 11) + 2; + if (grid_height > block_height) + continue; + + const uint32_t cem_index = s_unique_ldr_index_to_astc_cem[unique_cem_index]; + +#if defined(_DEBUG) || defined(DEBUG) + { + // Ensure configuration is actually valid. + astc_helpers::log_astc_block log_block; + log_block.clear(); + log_block.m_grid_width = (uint8_t)grid_width; + log_block.m_grid_height = (uint8_t)grid_height; + log_block.m_num_partitions = (uint8_t)(num_subsets + 1); + log_block.m_dual_plane = (ccs_index != 0); + log_block.m_color_component_selector = (uint8_t)(ccs_index ? (ccs_index - 1) : 0); + log_block.m_num_partitions = (uint8_t)(num_subsets + 1); + log_block.m_endpoint_ise_range = (uint8_t)(endpoint_ise_range + astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE); + log_block.m_weight_ise_range = (uint8_t)(weight_ise_range + astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE); + + for (uint32_t i = 0; i < log_block.m_num_partitions; i++) + log_block.m_color_endpoint_modes[i] = (uint8_t)cem_index; + + astc_helpers::astc_block phys_block; + bool pack_success = astc_helpers::pack_astc_block(phys_block, log_block, nullptr, nullptr, astc_helpers::cValidateSkipFinalEndpointWeightPacking); + assert(pack_success); + } +#endif + + const uint32_t tm_index = encoder_trial_modes.size_u32(); + + trial_mode& tm = *encoder_trial_modes.enlarge(1); + + tm.m_ccs_index = (int)ccs_index - 1; + tm.m_cem = cem_index; + tm.m_endpoint_ise_range = endpoint_ise_range + astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE; + tm.m_weight_ise_range = weight_ise_range; + tm.m_grid_width = grid_width; + tm.m_grid_height = grid_height; + tm.m_num_parts = num_subsets + 1; + + grouped_encoder_trial_modes.add(block_width, block_height, tm, tm_index); + + if (print_modes) + { + max_grid_width = basisu::maximum(max_grid_width, grid_width); + max_grid_height = basisu::maximum(max_grid_height, grid_height); + max_grid_samples = basisu::maximum(max_grid_samples, grid_width * grid_height); + + basisu::debug_printf("%u: CEM: %u DP: %u, CCS: %i, SUBSETS: %u, GRID: %ux%u, ENDPOINTS: %u, WEIGHTS: %u\n", + mode_index, + tm.m_cem, tm.m_ccs_index >= 0, tm.m_ccs_index, tm.m_num_parts, + tm.m_grid_width, tm.m_grid_height, + astc_helpers::get_ise_levels(tm.m_endpoint_ise_range), + astc_helpers::get_ise_levels(tm.m_weight_ise_range)); + } + + mode_index++; + } // cfg_index + + if (print_debug_info) + { + //fmt_debug_printf("create_encoder_trial_modes_table() time: {} secs\n", itm.get_elapsed_secs()); + basisu::debug_printf("create_encoder_trial_modes_table() - ASTC %ux%u modes\n", block_width, block_height); + basisu::debug_printf("Total used trial mode groups: %u\n", grouped_encoder_trial_modes.count_used_groups()); + basisu::debug_printf("Total ASTC configurations iterated: %u\n", mode_index); + if (print_modes) + basisu::fmt_debug_printf("Max grid dimensions: {}x{}, max grid samples: {}\n", max_grid_width, max_grid_height, max_grid_samples); + } + } + + // Cached encoder trial modes for each block size, to avoid having to compute this for every texture/mipmap level. + basisu::vector g_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + grouped_trial_modes g_grouped_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + grid_weight_dct g_grid_weight_dcts[astc_helpers::cTOTAL_BLOCK_SIZES]; + + // These tables could be initialized per transcoded texture, but that would result in per-texture overhead. + void init_transcoding_tables() + { + if (g_encoder_trial_modes[0].size()) + return; + + // We don't know what ASTC block sizes they're going to transcode, to prepare for all of them. + for (uint32_t i = 0; i < astc_helpers::cTOTAL_BLOCK_SIZES; i++) + { + const uint32_t block_width = astc_helpers::g_astc_block_sizes[i][0]; + const uint32_t block_height = astc_helpers::g_astc_block_sizes[i][1]; + + auto& encoder_trial_modes = g_encoder_trial_modes[i]; + auto& grouped_encoder_trial_modes = g_grouped_encoder_trial_modes[i]; + + encoder_trial_modes.reserve(3072); + create_encoder_trial_modes_table(block_width, block_height, encoder_trial_modes, grouped_encoder_trial_modes, false, false); + + g_grid_weight_dcts[i].init(block_width, block_height); + } // i + } + + const uint16_t g_total_unique_patterns[astc_helpers::NUM_ASTC_BLOCK_SIZES][2] = + { + { 437, 329 }, { 559, 405 }, { 659, 486 }, { 720, 534 }, + { 521, 333 }, { 584, 377 }, { 640, 410 }, { 672, 436 }, + { 710, 468 }, { 701, 476 }, { 759, 528 }, { 799, 568 }, + { 818, 597 }, { 838, 626 } + }; + + inline uint32_t get_total_unique_patterns(uint32_t astc_block_size_index, uint32_t num_parts) + { + assert(astc_block_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES); + assert((num_parts >= 2) && (num_parts <= 3)); + + return g_total_unique_patterns[astc_block_size_index][num_parts - 2]; + } + + const uint16_t g_unique_to_seed_4x4_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,33,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,68,69,70,71,72,73,74,75,76,78,83,87,89,90,91,94,95,98,99,100,101,107,108,109,110,111,113,114,115,116,119,121,122,124,125,128,129,130,131,134,135,137,138,139,142,143,144,146,147,149,150,151,156,158,159,161,165,167,168,169,170,171,172,174,175,177,181,183,184,191,194,195,196,198,199,203,204,206,207,208,210,211,213,214,215,216,218,220,222,226,227,230,231,232,235,236,239,245,246,247,248,249,250,252,253,254,255,257,258,260,262,264,270,271,273,277,278,279,280,281,284,291,293,299,302,304,305,306,307,309,314,319,324,325,326,327,329,330,335,337,339,341,343,344,347,348,351,352,354,355,359,362,368,370,373,374,375,376,380,386,387,388,389,394,395,399,404,409,411,412,418,419,422,423,426,430,432,438,441,443,445,447,453,455,463,471,474,475,476,478,479,484,487,488,489,490,491,495,496,498,500,504,510,511,513,517,518,523,524,526,527,529,530,531,534,539,542,546,547,549,553,558,567,578,581,583,586,587,591,593,594,595,598,600,601,602,605,607,611,612,614,615,619,622,625,627,631,633,634,638,639,643,647,649,655,658,661,662,663,664,666,672,673,674,681,683,684,686,690,693,694,695,696,700,703,705,707,713,716,719,720,724,726,727,730,731,732,736,742,751,754,756,762,764,766,769,770,773,774,778,780,789,791,796,798,799,801,802,804,807,810,811,812,818,819,821,826,828,831,833,834,836,839,840,842,847,849,852,868,872,873,877,881,886,887,888,890,895,897,898,899,902,903,906,911,914,915,919,923,924,930,934,937,938,943,945,947,948,950,951,954,958,959,963,964,966,967,971,976,983,987,988,993,994,995,998,999,1006,1007,1009,1013,1014,1015,1016,1019,1022,1023 } ; + const uint16_t g_unique_to_seed_5x4_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,78,83,87,88,89,90,91,94,95,97,98,99,100,101,107,108,109,110,111,113,114,115,116,119,121,122,124,125,128,129,130,131,132,134,135,137,138,139,142,143,144,145,146,147,149,150,151,153,154,156,157,158,159,161,165,167,168,169,170,171,172,174,175,177,181,183,184,185,188,190,191,194,195,196,198,199,200,203,204,206,207,208,210,211,213,214,215,216,217,218,220,222,225,226,227,229,230,231,232,235,236,239,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,265,267,270,271,273,275,277,278,279,280,281,282,284,287,291,293,295,296,299,300,302,304,305,306,307,309,314,317,319,323,324,325,326,327,329,330,332,335,337,339,341,342,343,344,347,348,349,350,351,352,354,355,359,361,362,365,368,370,373,374,375,376,380,381,386,387,388,389,391,394,395,399,404,405,407,409,410,411,412,418,419,420,422,423,426,430,432,438,439,441,443,445,447,449,453,454,455,462,463,465,471,473,474,475,476,478,479,482,484,486,487,488,489,490,491,495,496,498,500,501,503,504,505,508,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,542,546,547,549,550,551,553,554,558,563,567,569,572,575,578,579,581,583,586,587,591,593,594,595,598,600,601,602,605,606,607,608,611,612,614,615,616,619,622,623,625,627,631,633,634,636,638,639,643,645,647,649,652,655,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,690,692,693,694,695,696,697,700,702,703,705,707,709,711,713,716,719,720,724,725,726,727,730,731,732,736,739,742,748,751,754,756,758,762,763,764,766,768,769,770,772,773,774,776,778,780,782,786,789,791,792,796,798,799,801,802,804,807,810,811,812,814,818,819,821,823,826,828,830,831,833,834,835,836,839,840,842,845,847,849,852,858,861,866,868,870,871,872,873,876,877,878,881,886,887,888,890,891,895,897,898,899,901,902,903,906,909,911,914,915,919,923,924,927,929,930,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,958,959,963,964,966,967,970,971,975,976,980,983,986,987,988,993,994,995,997,998,999,1001,1006,1007,1009,1010,1013,1014,1015,1016,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_5x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,34,36,37,39,42,43,44,45,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,81,82,83,87,88,89,90,91,92,94,95,97,98,99,100,101,105,107,108,109,110,111,113,114,115,116,119,120,121,122,124,125,128,129,130,131,132,134,135,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,224,225,226,227,229,230,231,232,235,236,239,240,242,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,264,265,267,270,271,273,275,276,277,278,279,280,281,282,283,284,286,287,289,291,293,294,295,296,298,299,300,302,303,304,305,306,307,309,313,314,317,318,319,323,324,325,326,327,329,330,331,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,354,355,356,359,361,362,365,368,370,373,374,375,376,377,378,380,381,386,387,388,389,390,391,392,393,394,395,399,403,404,405,407,409,410,411,412,415,418,419,420,421,422,423,424,426,430,432,433,437,438,439,440,441,443,444,445,446,447,449,453,454,455,458,462,463,465,466,469,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,495,496,498,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,542,546,547,548,549,550,551,553,554,558,559,561,562,563,567,569,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,619,620,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,652,654,655,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,688,690,692,693,694,695,696,697,699,700,701,702,703,705,707,709,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,738,739,742,748,751,753,754,756,758,760,762,763,764,766,768,769,770,772,773,774,776,778,780,782,783,786,789,791,792,796,798,799,801,802,804,806,807,808,810,811,812,813,814,818,819,821,823,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,858,861,866,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,909,910,911,912,914,915,917,919,920,921,923,924,927,929,930,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,958,959,961,962,963,964,966,967,968,970,971,975,976,977,980,983,986,987,988,993,994,995,996,997,998,999,1001,1006,1007,1009,1010,1013,1014,1015,1016,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,20,21,23,24,25,26,27,28,29,30,31,33,34,36,37,39,42,43,44,45,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,86,87,88,89,90,91,92,94,95,97,98,99,100,101,105,107,108,109,110,111,112,113,114,115,116,117,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,220,222,223,224,225,226,227,229,230,231,232,235,236,239,240,242,243,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,264,265,266,267,270,271,273,275,276,277,278,279,280,281,282,283,284,286,287,289,291,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,317,318,319,323,324,325,326,327,329,330,331,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,354,355,356,357,359,360,361,362,363,365,367,368,370,371,373,374,375,376,377,378,380,381,383,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,420,421,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,453,454,455,458,461,462,463,465,466,469,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,493,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,532,533,534,536,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,564,567,569,570,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,651,652,654,655,657,658,661,662,663,664,665,666,668,672,673,674,675,681,683,684,686,687,688,690,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,742,745,747,748,751,753,754,756,758,760,762,763,764,766,768,769,770,772,773,774,776,778,780,782,783,784,786,788,789,791,792,795,796,798,799,801,802,804,806,807,808,810,811,812,813,814,818,819,820,821,823,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,856,858,861,866,868,869,870,871,872,873,874,875,876,877,878,879,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,906,908,909,910,911,912,914,915,917,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,970,971,972,975,976,977,979,980,982,983,986,987,988,993,994,995,996,997,998,999,1001,1004,1006,1007,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,14,15,16,17,18,19,21,23,24,25,26,27,28,29,31,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,114,115,116,119,121,122,125,128,129,130,131,132,134,135,136,137,138,139,142,144,146,147,148,149,151,153,154,156,158,159,167,168,169,170,171,172,174,175,177,178,181,183,190,191,193,194,195,196,198,199,203,206,207,210,211,213,214,215,216,217,218,220,222,223,225,226,227,229,230,231,232,235,236,237,240,247,249,250,252,254,255,257,258,260,262,264,266,267,270,271,272,273,277,278,279,281,283,284,286,289,291,292,293,295,298,299,302,303,305,306,307,309,314,316,318,319,323,324,326,327,329,335,339,341,343,347,348,351,352,353,354,355,361,362,363,366,367,368,370,373,374,376,380,386,387,388,389,390,392,393,394,395,402,403,404,407,409,411,414,415,418,419,422,423,426,430,432,433,436,437,438,439,440,441,445,447,450,452,453,455,458,461,463,470,471,474,475,476,478,480,482,486,487,488,490,495,496,498,499,500,504,506,508,510,513,517,518,519,521,523,524,526,527,529,530,531,533,534,538,539,543,545,546,547,549,550,551,553,554,558,562,567,569,572,578,581,583,585,586,587,591,593,594,595,596,598,600,601,602,606,607,608,610,611,612,614,615,616,619,621,622,623,625,627,631,633,634,638,639,643,645,646,647,648,649,652,655,658,661,662,663,664,665,670,672,673,674,675,681,683,684,685,686,687,688,690,691,693,694,695,696,697,700,702,703,705,713,714,716,719,720,722,724,726,727,730,731,732,736,740,742,751,753,754,756,758,760,762,763,764,766,769,770,774,778,780,786,789,791,794,797,798,799,801,802,804,805,807,808,810,811,812,817,819,821,822,826,828,831,834,835,836,839,842,847,852,854,858,862,868,869,870,872,874,876,877,878,886,887,890,891,893,895,897,898,899,902,903,906,909,910,911,912,914,919,929,930,934,936,937,938,942,943,945,947,948,950,951,954,955,958,959,963,964,966,967,968,976,980,983,987,988,993,994,995,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1019,1022,1023 }; + const uint16_t g_unique_to_seed_8x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,45,46,47,48,49,50,51,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,154,156,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,191,193,194,195,196,198,199,203,204,206,207,209,210,211,213,214,215,216,217,218,220,222,225,226,227,229,230,231,232,235,236,237,240,247,248,249,250,252,253,254,255,257,258,260,261,262,264,266,267,270,271,272,273,277,278,279,281,284,286,287,289,291,292,293,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,335,337,339,341,342,343,344,347,348,351,352,353,354,355,359,361,362,363,366,367,368,370,371,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,422,423,424,426,430,432,433,436,437,438,439,440,441,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,482,484,486,487,488,490,491,495,496,498,499,500,501,502,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,530,531,533,534,535,538,539,543,545,546,547,550,551,554,558,559,562,567,569,572,578,579,581,583,585,586,587,591,593,594,595,598,600,601,602,606,607,608,610,611,612,614,615,618,619,621,622,623,625,627,631,633,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,672,673,674,675,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,711,713,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,756,758,760,762,763,764,766,768,770,774,775,778,780,786,789,791,794,796,798,799,801,802,804,805,807,808,810,811,812,813,817,819,821,825,826,831,834,836,839,841,842,845,847,849,851,852,854,856,862,868,869,870,871,872,874,876,877,879,881,886,887,890,891,893,895,897,898,899,902,903,906,909,910,911,914,915,918,919,921,923,924,927,929,930,934,936,937,938,942,943,945,947,948,951,954,955,958,959,962,963,964,966,967,968,970,971,976,977,979,983,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1013,1015,1016,1018,1019,1022,1023 }; + const uint16_t g_unique_to_seed_8x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,153,154,156,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,223,225,226,227,229,230,231,232,235,236,237,240,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,266,267,270,271,272,273,277,278,279,281,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,335,337,339,341,342,343,344,347,348,349,350,351,352,353,354,355,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,414,415,418,419,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,480,482,484,486,487,488,489,490,491,495,496,498,499,500,501,502,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,543,545,546,547,549,550,551,553,554,558,559,562,567,569,572,575,578,579,581,583,585,586,587,591,593,594,595,596,598,600,601,602,606,607,608,610,611,612,614,615,616,618,619,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,756,758,760,762,763,764,766,768,769,770,773,774,775,776,778,780,786,789,791,792,794,796,797,798,799,801,802,804,805,807,808,810,811,812,813,817,818,819,821,822,825,826,828,830,831,833,834,835,836,839,841,842,845,847,849,851,852,854,856,858,861,862,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,895,897,898,899,902,903,906,909,910,911,912,914,915,918,919,921,923,924,927,929,930,933,934,935,936,937,938,940,942,943,945,947,948,950,951,954,955,958,959,962,963,964,966,967,968,970,971,976,977,979,980,983,986,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1018,1019,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x5_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,94,95,97,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,185,188,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,213,214,215,216,217,218,219,220,222,223,225,226,227,229,230,231,232,235,236,237,238,239,240,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,275,277,278,279,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,332,335,337,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,415,418,419,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,510,511,513,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,550,551,554,558,559,562,567,569,570,571,572,575,578,579,580,581,583,585,586,587,591,593,594,595,597,598,600,601,602,606,607,608,609,610,611,612,614,615,616,617,618,619,621,622,623,624,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,711,713,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,755,756,757,758,760,762,763,764,766,768,770,773,774,775,776,778,780,786,789,791,794,795,796,798,799,801,802,804,805,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,834,835,836,839,841,842,845,847,849,851,852,854,856,862,868,869,870,871,872,873,874,876,877,878,879,881,883,886,887,888,890,891,893,895,897,898,899,901,902,903,906,909,910,911,914,915,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,941,942,943,945,947,948,950,951,954,955,956,958,959,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,996,997,998,999,1001,1006,1007,1008,1009,1013,1014,1015,1016,1017,1018,1019,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x6_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,94,95,97,98,99,100,101,103,107,108,109,110,112,113,114,115,116,119,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,153,154,155,156,157,158,159,165,167,168,169,170,171,172,174,175,177,178,181,183,184,185,188,190,191,193,194,195,196,198,199,200,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,225,226,227,229,230,231,232,235,236,237,238,239,240,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,277,278,279,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,314,316,317,318,319,323,324,325,326,327,329,330,332,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,402,403,404,405,407,409,410,411,412,414,415,418,419,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,567,569,570,571,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,597,598,600,601,602,606,607,608,609,610,611,612,614,615,616,617,618,619,621,622,623,624,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,782,783,786,788,789,791,792,794,795,796,797,798,799,801,802,804,805,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,833,834,835,836,839,841,842,845,847,849,851,852,854,856,858,861,862,868,869,870,871,872,873,874,876,877,878,879,881,883,886,887,888,890,891,893,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_8x8_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,36,37,39,42,43,44,45,46,47,48,49,50,51,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,86,87,88,89,90,91,94,95,97,98,99,100,101,102,103,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,146,147,148,149,150,151,152,153,154,155,156,158,159,161,165,167,168,169,170,171,172,174,175,177,178,181,182,183,184,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,213,214,215,216,217,218,220,222,223,224,225,226,227,229,230,231,232,235,236,237,239,240,242,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,264,265,266,267,270,271,272,273,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,314,316,317,318,319,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,359,361,362,363,365,366,367,368,370,371,373,374,375,376,380,381,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,418,419,420,422,423,424,426,430,432,433,436,437,438,439,440,441,443,444,445,446,447,450,451,452,453,454,455,458,461,462,463,465,466,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,495,496,498,499,500,501,502,503,504,505,506,508,510,511,513,516,517,518,519,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,563,567,569,572,575,578,579,580,581,583,585,586,587,590,591,593,594,595,596,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,618,619,620,621,622,623,625,627,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,700,701,702,703,705,707,709,711,713,714,716,719,720,722,724,725,726,727,730,731,732,736,739,740,742,743,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,782,783,786,789,791,792,793,794,796,797,798,799,801,802,804,805,806,807,808,810,811,812,813,814,817,818,819,821,822,825,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,854,856,858,861,862,866,868,869,870,871,872,873,874,876,877,878,879,881,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,916,917,918,919,921,923,924,927,929,930,933,934,935,936,937,938,940,942,943,945,947,948,949,950,951,954,955,956,958,959,962,963,964,966,967,968,969,970,971,975,976,977,979,980,983,986,987,988,989,993,994,995,997,998,999,1001,1006,1007,1008,1009,1010,1013,1014,1015,1016,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x8_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,105,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,245,246,247,248,249,250,252,253,254,255,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,358,359,361,362,363,365,366,367,368,370,371,373,374,375,376,378,380,381,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,418,419,420,422,423,424,426,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,516,517,518,519,520,521,523,524,526,527,529,530,531,533,534,535,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,562,563,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,655,657,658,659,661,662,663,664,665,666,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,772,773,774,775,776,778,780,781,782,783,786,788,789,791,792,793,794,795,796,797,798,799,801,802,804,805,806,807,808,810,811,812,813,814,817,818,819,821,822,823,825,826,828,830,831,833,834,835,836,839,840,841,842,845,847,849,851,852,854,856,858,861,862,866,868,869,870,871,872,873,874,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,897,898,899,901,902,903,906,908,909,910,911,912,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_10x10_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,105,107,108,109,110,111,112,113,114,115,116,118,119,120,121,122,123,124,125,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,566,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,654,655,657,658,659,661,662,663,664,665,666,667,668,670,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,786,788,789,791,792,793,794,795,796,797,798,799,801,802,803,804,805,806,807,808,810,811,812,813,814,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,839,840,841,842,845,846,847,848,849,851,852,854,855,856,858,861,862,866,867,868,869,870,871,872,873,874,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x10_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,104,105,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,243,244,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,383,385,386,387,388,389,390,391,392,393,394,395,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,515,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,537,538,539,542,543,545,546,547,548,549,550,551,553,554,558,559,561,562,563,564,566,567,569,570,571,572,575,578,579,580,581,583,584,585,586,587,589,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,643,645,646,647,648,649,650,651,652,654,655,657,658,659,661,662,663,664,665,666,667,668,670,671,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,719,720,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,785,786,787,788,789,791,792,793,794,795,796,797,798,799,801,802,803,804,805,806,807,808,810,811,812,813,814,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,839,840,841,842,845,846,847,848,849,851,852,854,855,856,857,858,861,862,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,913,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,944,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x12_p2[] = { 1,2,3,4,5,7,8,9,10,11,13,14,15,16,17,18,19,20,21,23,24,25,26,27,28,29,30,31,32,33,34,36,37,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,94,95,97,98,99,100,101,102,103,104,105,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,128,129,130,131,132,134,135,136,137,138,139,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,161,165,167,168,169,170,171,172,174,175,176,177,178,179,180,181,182,183,184,185,186,188,190,191,193,194,195,196,198,199,200,201,202,203,204,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,229,230,231,232,235,236,237,238,239,240,242,243,244,245,246,247,248,249,250,252,253,254,255,256,257,258,260,261,262,263,264,265,266,267,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,286,287,289,290,291,292,293,294,295,296,298,299,300,302,303,304,305,306,307,309,310,313,314,316,317,318,319,321,323,324,325,326,327,329,330,331,332,333,335,337,338,339,340,341,342,343,344,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,365,366,367,368,370,371,373,374,375,376,377,378,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,399,400,402,403,404,405,407,409,410,411,412,414,415,417,418,419,420,421,422,423,424,425,426,427,428,429,430,432,433,436,437,438,439,440,441,443,444,445,446,447,449,450,451,452,453,454,455,456,458,459,461,462,463,465,466,469,470,471,473,474,475,476,478,479,480,481,482,484,486,487,488,489,490,491,492,493,494,495,496,498,499,500,501,502,503,504,505,506,508,509,510,511,513,514,515,516,517,518,519,520,521,522,523,524,526,527,528,529,530,531,532,533,534,535,536,537,538,539,542,543,545,546,547,548,549,550,551,553,554,557,558,559,561,562,563,564,566,567,569,570,571,572,575,576,578,579,580,581,583,584,585,586,587,589,590,591,593,594,595,596,597,598,599,600,601,602,605,606,607,608,609,610,611,612,614,615,616,617,618,619,620,621,622,623,624,625,627,628,629,631,633,634,636,638,639,640,643,644,645,646,647,648,649,650,651,652,654,655,657,658,659,660,661,662,663,664,665,666,667,668,670,671,672,673,674,675,681,683,684,685,686,687,688,690,691,692,693,694,695,696,697,699,700,701,702,703,705,706,707,709,711,713,714,716,717,719,720,721,722,723,724,725,726,727,730,731,732,736,738,739,740,742,743,745,747,748,751,753,754,755,756,757,758,760,762,763,764,766,768,769,770,771,772,773,774,775,776,778,780,781,782,783,784,785,786,787,788,789,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,810,811,812,813,814,815,817,818,819,820,821,822,823,825,826,828,829,830,831,833,834,835,836,837,839,840,841,842,844,845,846,847,848,849,851,852,854,855,856,857,858,861,862,863,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,883,886,887,888,890,891,893,894,895,896,897,898,899,901,902,903,904,906,908,909,910,911,912,913,914,915,916,917,918,919,920,921,923,924,927,929,930,932,933,934,935,936,937,938,940,941,942,943,944,945,947,948,949,950,951,954,955,956,958,959,960,961,962,963,964,966,967,968,969,970,971,972,975,976,977,979,980,982,983,986,987,988,989,993,994,995,996,997,998,999,1001,1004,1006,1007,1008,1009,1010,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023 }; + + const uint16_t g_unique_to_seed_4x4_p3[] = { 0,3,8,11,14,15,17,26,29,30,31,32,33,36,38,43,44,47,49,51,55,56,57,59,67,70,74,76,79,81,82,88,89,90,100,104,108,110,111,117,122,126,127,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,230,236,247,249,250,252,254,260,262,263,266,272,273,276,291,292,294,297,302,309,310,313,314,318,319,324,327,328,330,331,335,337,346,355,356,357,358,363,365,368,378,381,384,386,388,390,391,392,397,398,401,410,411,417,419,427,431,437,439,440,446,451,455,457,458,459,460,462,464,467,468,471,472,474,475,477,479,483,487,488,493,495,496,497,502,503,504,511,512,516,518,519,523,525,530,532,538,543,544,546,547,549,550,551,553,554,562,567,568,570,571,578,579,581,582,588,589,590,593,594,600,601,606,611,613,623,624,625,630,637,638,645,646,648,650,651,658,659,662,666,669,670,678,683,686,688,691,694,696,699,700,701,703,704,707,713,715,717,719,722,724,725,727,730,731,735,738,739,745,750,751,758,759,760,766,775,776,779,783,784,785,786,787,788,798,799,802,804,805,807,808,809,812,821,822,823,825,827,831,835,837,838,842,844,845,846,848,853,854,858,859,860,866,884,888,892,894,898,902,906,907,915,918,922,923,925,927,931,932,937,938,940,943,945,953,955,958,959,963,971,974,977,979,989,990,998,1005,1006,1007,1011,1012,1015,1020,1023 }; + const uint16_t g_unique_to_seed_5x4_p3[] = { 0,3,7,8,11,12,14,15,17,18,26,29,30,31,32,33,34,36,38,39,43,44,47,49,51,55,56,57,59,62,63,67,70,74,76,79,81,82,88,89,90,91,100,103,104,108,110,111,117,122,123,126,127,132,133,134,135,136,139,144,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,230,236,246,247,249,250,252,254,259,260,262,263,266,269,272,273,274,275,276,291,292,293,294,297,302,306,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,346,355,356,357,358,359,363,365,368,377,378,381,384,386,388,390,391,392,394,397,398,401,407,410,411,417,419,427,430,431,437,439,440,446,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,483,485,487,488,493,495,496,497,501,502,503,504,506,508,510,511,512,515,516,518,519,521,523,524,525,530,532,538,541,543,544,546,547,549,550,551,552,553,554,562,567,568,570,571,577,578,579,581,582,588,589,590,593,594,595,600,601,603,606,609,611,613,623,624,625,630,632,637,638,639,645,646,648,650,651,654,658,659,662,666,669,670,678,679,683,685,686,688,691,694,696,699,700,701,703,704,707,713,715,717,719,722,724,725,727,730,731,732,735,738,739,742,745,746,749,750,751,758,759,760,766,769,773,775,776,779,783,784,785,786,787,788,791,793,798,799,802,804,805,806,807,808,809,812,813,821,822,823,825,827,831,835,837,838,839,842,844,845,846,848,853,854,858,859,860,866,873,876,877,884,887,888,892,894,898,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,951,953,955,958,959,963,971,972,974,977,979,982,983,989,990,991,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_5x5_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,26,27,29,30,31,32,33,34,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,100,103,104,106,108,110,111,115,117,122,123,126,127,128,130,132,133,134,135,136,139,144,147,150,151,152,156,157,158,162,163,166,167,168,169,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,209,210,214,216,220,222,227,230,235,236,246,247,249,250,252,254,257,259,260,262,263,266,269,272,273,274,275,276,279,282,291,292,293,294,295,297,302,306,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,342,345,346,353,355,356,357,358,359,363,364,365,368,371,374,377,378,381,384,386,387,388,390,391,392,394,397,398,399,401,407,410,411,417,419,427,430,431,437,438,439,440,443,446,451,455,456,457,458,459,460,462,463,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,501,502,503,504,506,508,510,511,512,515,516,518,519,521,522,523,524,525,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,562,567,568,570,571,577,578,579,581,582,586,588,589,590,593,594,595,600,601,602,603,606,609,610,611,613,618,623,624,625,626,630,632,637,638,639,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,691,694,696,698,699,700,701,703,704,707,708,713,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,745,746,747,748,749,750,751,753,758,759,760,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,811,812,813,821,822,823,825,827,831,835,837,838,839,840,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,884,886,887,888,892,894,897,898,902,905,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,951,953,954,955,958,959,963,971,972,973,974,977,978,979,982,983,989,990,991,992,998,999,1004,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_6x5_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,21,23,26,27,29,30,31,32,33,34,35,36,38,39,42,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,130,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,162,163,166,167,168,169,171,173,175,176,179,181,182,183,186,189,192,198,199,200,203,205,207,209,210,214,216,220,222,227,230,231,235,236,245,246,247,249,250,252,254,257,259,260,262,263,266,269,272,273,274,275,276,279,281,282,288,291,292,293,294,295,297,300,302,306,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,342,345,346,348,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,397,398,399,401,407,410,411,412,413,417,419,427,430,431,437,438,439,440,443,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,508,510,511,512,513,515,516,518,519,521,522,523,524,525,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,566,567,568,570,571,577,578,579,580,581,582,584,586,588,589,590,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,626,630,632,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,711,713,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,884,886,887,888,892,894,897,898,900,902,905,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,951,953,954,955,957,958,959,963,967,971,972,973,974,977,978,979,982,983,986,989,990,991,992,998,999,1003,1004,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + + const uint16_t g_unique_to_seed_6x6_p3[] = { 0,8,11,14,15,17,18,19,26,31,34,35,36,38,44,47,48,49,51,56,59,61,70,74,76,82,88,90,96,100,103,104,108,110,111,117,122,123,126,127,132,133,135,139,147,150,151,152,156,157,163,166,168,171,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,247,249,250,252,254,260,261,262,263,266,272,273,275,276,288,291,292,293,294,297,302,309,310,313,314,318,327,328,331,335,337,346,356,357,358,363,365,368,378,381,384,386,390,391,392,396,397,398,399,401,410,411,419,427,430,431,437,439,440,451,455,457,458,459,460,462,468,470,471,472,474,475,477,479,482,483,488,493,495,496,502,503,504,507,510,511,512,515,516,518,519,522,523,525,526,527,538,543,544,546,547,549,550,552,553,554,562,570,578,579,581,582,588,589,590,593,595,600,606,611,613,618,623,625,632,637,638,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,691,694,696,698,699,700,701,703,704,707,713,714,715,717,719,722,724,727,730,731,734,738,739,743,747,748,750,751,753,758,760,764,766,769,775,776,783,784,785,787,791,793,798,799,802,804,805,806,807,808,809,810,813,822,823,825,831,835,837,838,839,840,842,845,846,848,853,854,858,859,860,866,874,882,884,887,888,892,894,898,902,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,953,955,958,959,963,966,971,974,979,990,991,998,999,1007,1010,1011,1012,1015,1020,1023 }; + const uint16_t g_unique_to_seed_8x5_p3[] = { 0,3,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,67,70,76,79,81,82,88,89,90,96,100,103,104,108,110,111,117,122,123,126,127,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,227,230,247,249,250,254,260,261,262,263,266,272,273,275,276,279,288,291,292,293,294,297,302,307,309,310,313,314,315,318,319,327,328,331,335,337,346,355,356,357,358,359,363,365,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,487,488,493,495,496,502,503,504,507,508,511,512,515,516,518,519,522,523,524,526,527,538,543,544,547,549,550,552,553,554,557,562,568,570,578,579,581,582,588,589,590,593,595,600,602,603,606,609,611,613,614,624,625,632,637,638,639,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,696,699,700,701,703,704,707,712,713,715,717,719,722,724,727,730,731,734,738,739,743,745,747,750,751,758,759,760,763,764,766,769,771,775,776,779,781,783,784,785,787,791,793,798,799,802,804,805,806,807,809,810,812,813,822,823,825,831,835,837,838,840,842,844,845,846,848,853,854,858,859,860,866,873,876,882,884,887,888,892,894,895,898,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,955,958,959,963,966,971,974,977,979,983,989,990,991,998,999,1005,1007,1010,1011,1012,1015,1023 }; + const uint16_t g_unique_to_seed_8x6_p3[] = { 0,3,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,67,70,74,76,79,81,82,88,89,90,96,100,103,104,108,110,111,117,122,123,126,127,131,132,133,134,135,139,147,150,151,152,156,157,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,227,230,236,247,249,250,252,254,260,261,262,263,266,272,273,275,276,279,288,291,292,293,294,297,302,307,309,310,313,314,315,318,319,324,327,328,331,335,337,338,346,355,356,357,358,359,363,365,368,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,485,487,488,493,495,496,502,503,504,507,508,510,511,512,515,516,518,519,522,523,524,525,526,527,538,541,543,544,546,547,549,550,552,553,554,557,562,566,567,568,570,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,623,624,625,632,637,638,639,645,646,650,651,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,738,739,743,745,747,748,750,751,753,758,759,760,763,764,766,769,771,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,808,809,810,812,813,822,823,825,831,835,837,838,839,840,842,844,845,846,848,850,853,854,858,859,860,866,873,874,876,882,884,887,888,892,894,895,898,900,902,906,907,914,915,918,919,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,955,958,959,963,966,971,974,977,979,983,989,990,991,998,999,1005,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x5_p3[] = { 0,3,7,8,11,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,62,67,70,72,76,79,81,82,88,89,90,91,95,96,100,103,104,108,110,111,114,117,122,123,126,127,131,132,133,134,135,139,140,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,213,214,216,227,230,245,247,249,250,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,307,309,310,313,314,315,318,319,327,328,331,335,337,346,355,356,357,358,359,363,365,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,412,413,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,487,488,493,495,496,500,501,502,503,504,506,507,508,510,511,512,515,516,518,519,521,522,523,524,526,527,530,538,541,543,544,547,549,550,552,553,554,555,557,562,565,568,570,577,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,624,625,632,637,638,639,645,646,650,651,654,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,712,713,715,717,719,722,724,725,727,730,731,732,734,738,739,742,743,745,747,749,750,751,758,759,760,763,764,765,766,769,771,773,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,809,810,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,848,853,854,858,859,860,866,869,873,876,877,882,884,887,888,891,892,894,895,898,900,902,905,906,907,909,914,915,918,919,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,974,975,977,978,979,983,989,990,991,993,998,999,1005,1007,1010,1011,1012,1015,1023 }; + + const uint16_t g_unique_to_seed_10x6_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,31,33,34,35,36,38,43,44,47,48,49,51,55,56,59,61,62,67,70,72,74,76,79,81,82,88,89,90,91,95,96,100,103,104,108,110,111,114,117,122,123,126,127,131,132,133,134,135,139,140,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,179,181,182,183,186,189,192,199,203,205,207,210,213,214,216,222,227,230,236,245,246,247,249,250,252,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,346,355,356,357,358,359,363,364,365,368,377,378,381,384,386,390,391,392,394,396,397,398,399,401,407,410,411,412,413,419,424,427,430,431,437,439,440,450,451,455,457,458,459,460,462,464,467,468,470,471,472,474,475,477,478,479,482,483,485,487,488,493,495,496,500,501,502,503,504,506,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,538,539,541,543,544,546,547,549,550,552,553,554,555,557,562,565,566,567,568,570,577,578,579,581,582,588,589,590,593,595,600,601,602,603,606,609,611,613,614,618,623,624,625,632,637,638,639,645,646,648,650,651,654,658,659,662,666,667,669,670,678,679,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,747,748,749,750,751,753,758,759,760,763,764,765,766,769,771,773,775,776,779,781,783,784,785,786,787,791,793,798,799,802,804,805,806,807,808,809,810,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,848,850,853,854,858,859,860,866,869,873,874,876,877,882,884,887,888,891,892,894,895,898,900,902,905,906,907,909,914,915,918,919,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,974,975,977,978,979,982,983,989,990,991,993,998,999,1005,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_8x8_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,63,67,70,72,74,76,79,81,82,88,89,90,96,100,103,104,106,108,110,111,117,122,123,126,127,131,132,133,134,135,136,139,144,147,150,151,152,156,157,158,163,166,167,168,171,173,175,176,178,179,181,182,183,186,189,192,199,203,205,207,210,214,216,222,227,230,235,236,246,247,249,250,252,254,260,261,262,263,266,269,272,273,275,276,279,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,342,345,346,355,356,357,358,359,363,365,368,371,377,378,381,384,386,388,390,391,392,394,396,397,398,399,401,407,410,411,417,419,424,427,430,431,437,439,440,446,450,451,455,457,458,459,460,462,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,502,503,504,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,557,562,566,567,568,570,571,577,578,579,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,630,632,637,638,639,645,646,648,650,651,658,659,662,666,667,669,670,671,678,679,683,685,686,687,688,689,691,694,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,743,745,746,747,748,750,751,753,758,759,760,763,764,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,837,838,839,840,842,844,845,846,847,848,850,852,853,854,858,859,860,866,873,874,876,877,882,884,886,887,888,892,894,895,897,898,900,902,906,907,914,915,918,919,920,922,923,925,927,931,932,937,938,940,943,944,945,947,951,953,954,955,958,959,963,966,971,972,974,977,979,982,983,989,990,991,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023 }; + const uint16_t g_unique_to_seed_10x8_p3[] = { 0,3,7,8,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,76,79,81,82,88,89,90,91,94,95,96,100,103,104,106,108,110,111,114,115,117,122,123,126,127,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,163,166,167,168,171,173,175,176,178,179,181,182,183,186,189,192,198,199,203,205,207,210,213,214,216,220,222,227,230,235,236,245,246,247,249,250,252,254,259,260,261,262,263,266,269,272,273,274,275,276,279,281,288,291,292,293,294,295,297,302,306,307,309,310,311,313,314,315,318,319,324,327,328,330,331,335,337,338,342,345,346,355,356,357,358,359,363,364,365,368,371,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,410,411,412,413,417,419,424,427,430,431,437,438,439,440,443,446,450,451,455,457,458,459,460,462,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,562,565,566,567,568,570,571,577,578,579,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,630,632,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,669,670,671,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,982,983,986,989,990,991,993,998,999,1005,1006,1007,1010,1011,1012,1015,1020,1022,1023}; + const uint16_t g_unique_to_seed_10x10_p3[] = { 0,3,7,8,10,11,12,14,15,17,18,19,23,26,27,29,30,31,32,33,34,35,36,38,39,40,43,44,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,75,76,79,81,82,88,89,90,91,94,95,96,100,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,130,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,162,163,166,167,168,169,171,173,175,176,178,179,181,182,183,186,189,192,198,199,200,203,205,207,209,210,213,214,216,218,220,222,227,230,235,236,238,242,245,246,247,249,250,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,288,291,292,293,294,295,297,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,417,419,424,427,430,431,435,437,438,439,440,443,446,450,451,455,456,457,458,459,460,462,463,464,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,562,565,566,567,568,570,571,577,578,579,580,581,582,586,588,589,590,592,593,594,595,600,601,602,603,606,609,610,611,613,614,618,623,624,625,626,630,632,634,637,638,639,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,825,827,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,998,999,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1022,1023 }; + + const uint16_t g_unique_to_seed_12x10_p3[] = { 0,3,7,8,10,11,12,14,15,16,17,18,19,21,23,26,27,29,30,31,32,33,34,35,36,38,39,40,42,43,44,45,47,48,49,50,51,55,56,57,59,60,61,62,63,67,70,72,74,75,76,79,81,82,88,89,90,91,94,95,96,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,129,130,131,132,133,134,135,136,139,140,144,147,150,151,152,153,156,157,158,161,162,163,166,167,168,169,171,173,175,176,178,179,181,182,183,185,186,189,192,195,198,199,200,203,205,207,209,210,213,214,216,218,220,222,227,230,231,235,236,238,242,245,246,247,249,250,251,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,283,288,291,292,293,294,295,297,300,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,348,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,415,417,419,424,427,430,431,435,437,438,439,440,443,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,538,539,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,563,565,566,567,568,570,571,577,578,579,580,581,582,584,586,588,589,590,592,593,594,595,600,601,602,603,604,606,609,610,611,613,614,618,623,624,625,626,630,632,634,637,638,639,643,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,673,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,711,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,769,771,773,775,776,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,828,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,914,915,918,919,920,922,923,925,927,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,995,998,999,1002,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1021,1022,1023 }; + const uint16_t g_unique_to_seed_12x12_p3[] = { 0,3,4,7,8,10,11,12,14,15,16,17,18,19,21,23,26,27,29,30,31,32,33,34,35,36,38,39,40,42,43,44,45,47,48,49,50,51,53,55,56,57,58,59,60,61,62,63,67,70,72,74,75,76,79,81,82,83,88,89,90,91,94,95,96,100,102,103,104,106,108,110,111,114,115,117,120,122,123,126,127,128,129,130,131,132,133,134,135,136,138,139,140,144,147,150,151,152,153,156,157,158,159,160,161,162,163,166,167,168,169,171,173,175,176,177,178,179,181,182,183,185,186,189,192,195,196,198,199,200,203,205,207,208,209,210,213,214,216,218,220,222,227,230,231,235,236,238,242,245,246,247,249,250,251,252,254,257,259,260,261,262,263,266,269,272,273,274,275,276,279,281,282,283,288,291,292,293,294,295,297,300,302,306,307,308,309,310,311,313,314,315,318,319,324,326,327,328,330,331,335,337,338,342,345,346,347,348,350,353,355,356,357,358,359,363,364,365,368,371,372,374,377,378,379,381,384,386,387,388,390,391,392,394,395,396,397,398,399,401,407,408,410,411,412,413,415,417,419,424,426,427,430,431,432,435,437,438,439,440,443,444,446,450,451,455,456,457,458,459,460,461,462,463,464,465,466,467,468,470,471,472,474,475,477,478,479,480,482,483,485,487,488,493,495,496,497,500,501,502,503,504,505,506,507,508,510,511,512,513,515,516,518,519,521,522,523,524,525,526,527,530,532,535,538,539,540,541,543,544,546,547,549,550,551,552,553,554,555,557,558,562,563,565,566,567,568,569,570,571,577,578,579,580,581,582,584,586,588,589,590,592,593,594,595,600,601,602,603,604,606,609,610,611,613,614,618,623,624,625,626,628,630,631,632,634,636,637,638,639,640,643,644,645,646,648,650,651,654,658,659,662,666,667,668,669,670,671,673,678,679,683,685,686,687,688,689,691,694,695,696,698,699,700,701,703,704,707,708,711,712,713,714,715,717,719,722,724,725,727,730,731,732,734,735,738,739,742,743,745,746,747,748,749,750,751,753,758,759,760,763,764,765,766,767,768,769,771,773,774,775,776,778,779,780,781,783,784,785,786,787,788,789,790,791,793,794,798,799,800,802,804,805,806,807,808,809,810,811,812,813,821,822,823,824,825,827,828,831,835,836,837,838,839,840,841,842,843,844,845,846,847,848,850,852,853,854,858,859,860,863,866,869,873,874,876,877,881,882,884,886,887,888,891,892,894,895,897,898,900,902,905,906,907,909,911,912,914,915,918,919,920,922,923,925,927,929,930,931,932,937,938,939,940,943,944,945,947,951,952,953,954,955,957,958,959,961,963,966,967,971,972,973,974,975,977,978,979,980,982,983,986,989,990,991,992,993,995,998,999,1000,1002,1003,1004,1005,1006,1007,1010,1011,1012,1014,1015,1020,1021,1022,1023 }; + + static const uint16_t* g_unique_index_to_astc_part_seed[2][astc_helpers::NUM_ASTC_BLOCK_SIZES] = // [num_parts][astc_block_size_index] + { + { + g_unique_to_seed_4x4_p2, g_unique_to_seed_5x4_p2, g_unique_to_seed_5x5_p2, g_unique_to_seed_6x5_p2, + g_unique_to_seed_6x6_p2, g_unique_to_seed_8x5_p2, g_unique_to_seed_8x6_p2, g_unique_to_seed_10x5_p2, + g_unique_to_seed_10x6_p2, g_unique_to_seed_8x8_p2, g_unique_to_seed_10x8_p2, g_unique_to_seed_10x10_p2, + g_unique_to_seed_12x10_p2, g_unique_to_seed_12x12_p2 + }, + { + g_unique_to_seed_4x4_p3, g_unique_to_seed_5x4_p3, g_unique_to_seed_5x5_p3, g_unique_to_seed_6x5_p3, + g_unique_to_seed_6x6_p3, g_unique_to_seed_8x5_p3, g_unique_to_seed_8x6_p3, g_unique_to_seed_10x5_p3, + g_unique_to_seed_10x6_p3, g_unique_to_seed_8x8_p3, g_unique_to_seed_10x8_p3, g_unique_to_seed_10x10_p3, + g_unique_to_seed_12x10_p3, g_unique_to_seed_12x12_p3 + } + }; + + static inline uint16_t unique_pat_index_to_part_seed(uint32_t astc_block_size_index, uint32_t num_parts, uint32_t unique_pat_index) + { + assert(astc_block_size_index < astc_helpers::NUM_ASTC_BLOCK_SIZES); + assert((num_parts >= 2) && (num_parts <= 3)); + assert(unique_pat_index < get_total_unique_patterns(astc_block_size_index, num_parts)); + + return g_unique_index_to_astc_part_seed[num_parts - 2][astc_block_size_index][unique_pat_index]; + } + + static bool zstd_decompress(const void *pComp_data, size_t comp_size, basisu::uint8_vec &uncomp_data) + { + if (!comp_size) + { + uncomp_data.resize(0); + return true; + } + +#if BASISD_SUPPORT_KTX2_ZSTD + const uint64_t decomp_size = ZSTD_getFrameContentSize(pComp_data, comp_size); + + if ((decomp_size == ZSTD_CONTENTSIZE_UNKNOWN) || (decomp_size == ZSTD_CONTENTSIZE_ERROR)) + { + BASISU_DEVEL_ERROR("zstd_decompress: ZSTD_getFrameContentSize failed\n"); + return false; + } + + // sanity check, not UINT32_MAX purposely, even INT_MAX is too high + if (decomp_size > (uint64_t)INT32_MAX) + { + BASISU_DEVEL_ERROR("zstd_decompress: decompressed size too large\n"); + return false; + } + + if (!uncomp_data.try_resize((size_t)decomp_size)) + { + BASISU_DEVEL_ERROR("zstd_decompress: Out of memory\n"); + return false; + } + + if (!decomp_size) + return true; + + const size_t actual_uncomp_size = ZSTD_decompress(uncomp_data.data(), uncomp_data.size(), pComp_data, comp_size); + if (ZSTD_isError(actual_uncomp_size)) + { + BASISU_DEVEL_ERROR("zstd_decompress: Zstd decompression failed, file is invalid or corrupted\n"); + return false; + } + + assert(actual_uncomp_size == decomp_size); + uncomp_data.resize(actual_uncomp_size); + + return true; +#else + BASISU_DEVEL_ERROR("zstd_decompress: file uses ZStd compression, but ZStd support disabled (see BASISD_SUPPORT_KTX2_ZSTD) \n"); + return false; +#endif + } + + static bool zstd_decompress_and_advance(const uint8_t* &pComp_data, size_t comp_size, basisu::uint8_vec& uncomp_data, simplified_bitwise_decoder& dec) + { + if (!zstd_decompress(pComp_data, comp_size, uncomp_data)) + return false; + pComp_data += comp_size; + dec.init(uncomp_data.data(), uncomp_data.size()); + return true; + } + + bool xuastc_ldr_decompress_image_full_zstd( + const uint8_t* pComp_data_all, size_t comp_data_size_all, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + if (comp_data_size_all < sizeof(xuastc_ldr_full_zstd_header)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const xuastc_ldr_full_zstd_header* pHdr = (const xuastc_ldr_full_zstd_header*)pComp_data_all; + + if ((!pHdr->m_raw_bits_len) || (!pHdr->m_mode_bytes_len)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint64_t total_comp_size = (uint64_t)((uint32_t)pHdr->m_raw_bits_len) + + pHdr->m_mode_bytes_len + pHdr->m_solid_dpcm_bytes_len + pHdr->m_endpoint_dpcm_reuse_indices_len + pHdr->m_use_bc_bits_len + + pHdr->m_endpoint_dpcm_3bit_len + pHdr->m_endpoint_dpcm_4bit_len + pHdr->m_endpoint_dpcm_5bit_len + pHdr->m_endpoint_dpcm_6bit_len + pHdr->m_endpoint_dpcm_7bit_len + pHdr->m_endpoint_dpcm_8bit_len + + pHdr->m_mean0_bits_len + pHdr->m_mean1_bytes_len + + pHdr->m_run_bytes_len + pHdr->m_coeff_bytes_len + pHdr->m_sign_bits_len + + pHdr->m_weight2_bits_len + pHdr->m_weight3_bits_len + pHdr->m_weight4_bits_len + pHdr->m_weight8_bytes_len; + + if (comp_data_size_all < (sizeof(xuastc_ldr_full_zstd_header) + total_comp_size)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + bitwise_decoder raw_bits; + simplified_bitwise_decoder comp_mode_dec, solid_dpcm_dec, endpoint_dpcm_reuse_indices_dec, use_bc_bits_dec; + simplified_bitwise_decoder endpoint_dpcm_3bit_dec, endpoint_dpcm_4bit_dec, endpoint_dpcm_5bit_dec, endpoint_dpcm_6bit_dec, endpoint_dpcm_7bit_dec, endpoint_dpcm_8bit_dec; + + basisu::uint8_vec uncomp_mode_bytes, uncomp_solid_dpcm_bytes, uncomp_endpoint_dpcm_reuse_indices, uncomp_use_bc_bits; + basisu::uint8_vec uncomp_endpoint_dpcm_3bit, uncomp_endpoint_dpcm_4bit, uncomp_endpoint_dpcm_5bit, uncomp_endpoint_dpcm_6bit, uncomp_endpoint_dpcm_7bit, uncomp_endpoint_dpcm_8bit; + + basisu::uint8_vec uncomp_mean0_bits, uncomp_mean1_bytes, uncomp_run_bytes, uncomp_coeff_bytes, uncomp_weight2_bytes, uncomp_weight3_bytes, uncomp_weight4_bytes, uncomp_weight8_bytes; + simplified_bitwise_decoder mean0_bits, mean1_bytes, run_bytes, coeff_bytes, sign_bits, weight2_bits, weight3_bits, weight4_bits, weight8_bytes; + + const uint8_t* pCur_buf = pComp_data_all + sizeof(xuastc_ldr_full_zstd_header); + + // raw bits + { + raw_bits.init(pCur_buf, pHdr->m_raw_bits_len); + pCur_buf += pHdr->m_raw_bits_len; + } + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mode_bytes_len, uncomp_mode_bytes, comp_mode_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_solid_dpcm_bytes_len, uncomp_solid_dpcm_bytes, solid_dpcm_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_reuse_indices_len, uncomp_endpoint_dpcm_reuse_indices, endpoint_dpcm_reuse_indices_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_use_bc_bits_len, uncomp_use_bc_bits, use_bc_bits_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_3bit_len, uncomp_endpoint_dpcm_3bit, endpoint_dpcm_3bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_4bit_len, uncomp_endpoint_dpcm_4bit, endpoint_dpcm_4bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_5bit_len, uncomp_endpoint_dpcm_5bit, endpoint_dpcm_5bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_6bit_len, uncomp_endpoint_dpcm_6bit, endpoint_dpcm_6bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_7bit_len, uncomp_endpoint_dpcm_7bit, endpoint_dpcm_7bit_dec)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_endpoint_dpcm_8bit_len, uncomp_endpoint_dpcm_8bit, endpoint_dpcm_8bit_dec)) + return false; + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mean0_bits_len, uncomp_mean0_bits, mean0_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_mean1_bytes_len, uncomp_mean1_bytes, mean1_bytes)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_run_bytes_len, uncomp_run_bytes, run_bytes)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_coeff_bytes_len, uncomp_coeff_bytes, coeff_bytes)) + return false; + + // sign + { + sign_bits.init(pCur_buf, pHdr->m_sign_bits_len); + pCur_buf += pHdr->m_sign_bits_len; + } + + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight2_bits_len, uncomp_weight2_bytes, weight2_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight3_bits_len, uncomp_weight3_bytes, weight3_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight4_bits_len, uncomp_weight4_bytes, weight4_bits)) + return false; + if (!zstd_decompress_and_advance(pCur_buf, pHdr->m_weight8_bytes_len, uncomp_weight8_bytes, weight8_bytes)) + return false; + + // sanity check + const uint64_t total_read_size = pCur_buf - pComp_data_all; + if (total_read_size > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint32_t header_val = raw_bits.get_bits(FULL_ZSTD_HEADER_MARKER_BITS); + if (header_val != FULL_ZSTD_HEADER_MARKER) + { + BASISU_DEVEL_ERROR("Invalid marker\n"); + return false; + } + + const uint32_t astc_block_size_index = raw_bits.get_bits(4); + if (astc_block_size_index >= astc_helpers::NUM_ASTC_BLOCK_SIZES) + { + BASISU_DEVEL_ERROR("Invalid block dimension index\n"); + return false; + } + + astc_block_width = astc_helpers::g_astc_block_sizes[astc_block_size_index][0]; + astc_block_height = astc_helpers::g_astc_block_sizes[astc_block_size_index][1]; + + uses_srgb_astc_decode_mode = raw_bits.get_bits(1); + + actual_width = raw_bits.get_bits(16); + actual_height = raw_bits.get_bits(16); + has_alpha = raw_bits.get_bits(1); + + const bool use_dct = (raw_bits.get_bits(1) != 0); + + int int_q = 0; + if (use_dct) + int_q = raw_bits.get_bits(8); + + const float dct_q = (float)int_q / 2.0f; + if ((use_dct) && ((dct_q <= 0.0f) || (dct_q > 100.0f))) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid DCT global quality factor\n"); + return false; + } + + if (debug_output) + { + basisu::fmt_debug_printf("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: block dim: {}x{}, image dim: {}x{}, sRGB decode profile: {}, has_alpha: {}, dct: {} dct_q: {}\n", + astc_block_width, astc_block_height, + actual_width, actual_height, + uses_srgb_astc_decode_mode, has_alpha, + use_dct, dct_q); + } + + const uint32_t num_blocks_x = (actual_width + astc_block_width - 1) / astc_block_width; + const uint32_t num_blocks_y = (actual_height + astc_block_height - 1) / astc_block_height; + + if (pInit_callback) + { + if (!(*pInit_callback)(num_blocks_x, num_blocks_y, astc_block_width, astc_block_height, uses_srgb_astc_decode_mode, dct_q, has_alpha, pInit_callback_data)) + return false; + } + + fvec dct_work; + + assert((size_t)astc_block_size_index < std::size(g_encoder_trial_modes)); + const auto& encoder_trial_modes = g_encoder_trial_modes[astc_block_size_index]; + + const grid_weight_dct& grid_dct = g_grid_weight_dcts[astc_block_size_index]; + + basisu::vector2D log_blocks; + if (!log_blocks.try_resize(num_blocks_x, 8)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: out of memory\n"); + return false; + } + + memset(log_blocks.get_ptr(), 0, log_blocks.size_in_bytes()); + + basisu::vector2D prev_block_states; + if (!prev_block_states.try_resize(num_blocks_x, 2)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: out of memory\n"); + return false; + } + + uint32_t cur_run_len = 0; + + int part2_hash[PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + PART_HASH_SIZE, -1); + + int part3_hash[PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + PART_HASH_SIZE, -1); + + int tm_hash[TM_HASH_SIZE]; + std::fill(tm_hash, tm_hash + TM_HASH_SIZE, -1); + + dct_syms syms; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + prev_block_state_full_zstd& new_prev_state = prev_block_states(bx, by & 1); + + const prev_block_state_full_zstd* pLeft_state = bx ? &prev_block_states(bx - 1, by & 1) : nullptr; + const prev_block_state_full_zstd* pUpper_state = by ? &prev_block_states(bx, (by - 1) & 1) : nullptr; + + astc_helpers::log_astc_block& log_blk = log_blocks(bx, by & 7); + + if (cur_run_len) + { + const prev_block_state_full_zstd* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + memcpy((void*)&log_blk, (const void*)&prev_log_blk, sizeof(log_blk)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + //new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + + cur_run_len--; + continue; + } + + const prev_block_state_full_zstd* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, (by - 1) & 1) : nullptr; + + // TODO: End check + const uint32_t mode_byte = comp_mode_dec.get_bits8(); + + if ((mode_byte & 3) == (uint32_t)xuastc_zstd_mode::cMODE_RUN) + { + // run + cur_run_len = 1 + (mode_byte >> 2); + + if (!bx && !by) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid run command\n"); + return false; + } + + const uint32_t max_possible_run_len = num_blocks_x - bx; + if (cur_run_len > max_possible_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid run len\n"); + return false; + } + + const prev_block_state_full_zstd* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + memcpy((void*)&log_blk, (const void*)&prev_log_blk, sizeof(log_blk)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + + cur_run_len--; + + continue; + } + else if ((mode_byte & 15) == (uint32_t)xuastc_zstd_mode::cMODE_SOLID) + { + // solid + const astc_helpers::log_astc_block* pPrev_log_blk = bx ? &log_blocks(bx - 1, by & 7) : (by ? &log_blocks(bx, (by - 1) & 7) : nullptr); + + uint32_t prev_solid_color[4] = { 0 }; + + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + uint32_t delta_r = solid_dpcm_dec.get_bits8(); + uint32_t delta_g = solid_dpcm_dec.get_bits8(); + uint32_t delta_b = solid_dpcm_dec.get_bits8(); + uint32_t delta_a = has_alpha ? solid_dpcm_dec.get_bits8() : 0; + + uint32_t r = (prev_solid_color[0] + delta_r) & 0xFF; + uint32_t g = (prev_solid_color[1] + delta_g) & 0xFF; + uint32_t b = (prev_solid_color[2] + delta_b) & 0xFF; + uint32_t a = 255; + if (has_alpha) + a = (prev_solid_color[3] + delta_a) & 0xFF; + + log_blk.clear(); + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | (r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | (g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | (b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | (a << 8)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_tm_index = -1; + + continue; + } + + new_prev_state.clear(); + + //log_blk.clear(); + memset((void*)&log_blk, 0, offsetof(astc_helpers::log_astc_block, m_weights)); + + uint32_t tm_index = 0; + uint32_t actual_cem = 0; + + if ((mode_byte & 1) == 0) + { + // raw + uint32_t config_reuse_index = (mode_byte >> 1) & 3; + + if (config_reuse_index < 3) + { + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state_full_zstd* pCfg_state = nullptr; + + switch (config_reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + new_prev_state.m_tm_index = tm_index; + //new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + } + else + { + if (mode_byte & XUASTC_LDR_MODE_BYTE_TM_HASH_HIT_FLAG) + { + uint32_t tm_hash_index = raw_bits.get_bits(TM_HASH_BITS); + tm_index = tm_hash[tm_hash_index]; + } + else + { + tm_index = raw_bits.decode_truncated_binary(encoder_trial_modes.size_u32()); + + tm_hash[tm_hash_index(tm_index)] = tm_index; + } + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid tm_index\n"); + return false; + } + + new_prev_state.m_tm_index = tm_index; + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + actual_cem = tm.m_cem; + + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // Decode is_base_ofs bit + bool is_base_ofs = (mode_byte & XUASTC_LDR_MODE_BYTE_IS_BASE_OFS_FLAG) != 0; + + if (is_base_ofs) + { + if (actual_cem == astc_helpers::CEM_LDR_RGB_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET; + else if (actual_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET; + } + } + + if (tm.m_num_parts > 1) + { + const uint32_t total_unique_indices = get_total_unique_patterns(astc_block_size_index, tm.m_num_parts); + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + const bool hash_hit_flag = (mode_byte & XUASTC_LDR_MODE_BYTE_PART_HASH_HIT) != 0; + + uint32_t unique_pat_index; + if (hash_hit_flag) + { + uint32_t h = raw_bits.get_bits(basist::astc_ldr_t::PART_HASH_BITS); + + unique_pat_index = pPart_hash[h]; + } + else + { + unique_pat_index = raw_bits.decode_truncated_binary(total_unique_indices); + + pPart_hash[basist::astc_ldr_t::part_hash_index(unique_pat_index)] = unique_pat_index; + } + + if (unique_pat_index >= get_total_unique_patterns(astc_block_size_index, tm.m_num_parts)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid unique_pat_index, decompression failed (file corrupt)\n"); + return false; + } + + log_blk.m_partition_id = unique_pat_index_to_part_seed(astc_block_size_index, tm.m_num_parts, unique_pat_index); + } + + + } // if (config_reuse_index < 3) + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + log_blk.m_color_endpoint_modes[part_iter] = (uint8_t)actual_cem; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + log_blk.m_dual_plane = (tm.m_ccs_index >= 0); + if (log_blk.m_dual_plane) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + const bool used_dpcm_endpoints_flag = (mode_byte & XUASTC_LDR_MODE_BYTE_DPCM_ENDPOINTS_FLAG) != 0; + + if (used_dpcm_endpoints_flag) + { + const int num_endpoint_levels = astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range); + const auto& endpoint_rank_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_rank_to_ISE; + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + uint32_t reuse_delta_index = endpoint_dpcm_reuse_indices_dec.get_bits8(); + if (reuse_delta_index >= NUM_REUSE_XY_DELTAS) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + const int reuse_bx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_x; + const int reuse_by = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_y; + + if ((reuse_bx < 0) || (reuse_by < 0) || (reuse_bx >= (int)num_blocks_x) || (reuse_by >= (int)num_blocks_y)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = &log_blocks(reuse_bx, reuse_by & 7); + if (pEndpoint_pred_log_blk->m_solid_color_flag_ldr) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid reuse delta\n"); + return false; + } + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { }; + + if (actual_cem_supports_bc) + { + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + endpoints_use_bc[part_iter] = (use_bc_bits_dec.get_bits1() != 0); + } + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + // Mini-CEM encoder, to cross CEM domains. + bool conv_status = convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + log_blk.m_color_endpoint_modes[0], log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Failed predicting endpoints\n"); + return false; + } + } + + if (num_endpoint_levels <= 8) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_3bit_dec.get_bits4(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 16) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_4bit_dec.get_bits4(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 32) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_5bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 64) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_6bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else if (num_endpoint_levels <= 128) + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_7bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + else + { + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = endpoint_dpcm_8bit_dec.get_bits8(); + + int e_val = (delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]]) % num_endpoint_levels; + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + } + } + } + } + else + { + if (!decode_values(raw_bits, tm.m_num_parts * total_endpoint_vals, log_blk.m_endpoint_ise_range, log_blk.m_endpoints)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: decode_values() failed\n"); + return false; + } + } + } + else if ((mode_byte & 15) >= (uint32_t)xuastc_zstd_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT) + { + // reuse full cfg+endpoints+part id + const uint32_t reuse_index = ((mode_byte >> 2) & 3) - 1; + + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state_full_zstd* pCfg_state = nullptr; + + switch (reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + const astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Invalid config reuse\n"); + return false; + } + + tm_index = pCfg_state->m_tm_index; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + for (uint32_t i = 0; i < cfg_log_blk.m_num_partitions; i++) + log_blk.m_color_endpoint_modes[i] = (uint8_t)actual_cem; + + log_blk.m_dual_plane = cfg_log_blk.m_dual_plane; + log_blk.m_color_component_selector = cfg_log_blk.m_color_component_selector; + log_blk.m_num_partitions = cfg_log_blk.m_num_partitions; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + log_blk.m_endpoint_ise_range = cfg_log_blk.m_endpoint_ise_range; + log_blk.m_weight_ise_range = cfg_log_blk.m_weight_ise_range; + log_blk.m_grid_width = cfg_log_blk.m_grid_width; + log_blk.m_grid_height = cfg_log_blk.m_grid_height; + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem) * log_blk.m_num_partitions; + memcpy(log_blk.m_endpoints, cfg_log_blk.m_endpoints, total_endpoint_vals); + + new_prev_state.m_tm_index = tm_index; + } + else + { + // shouldn't actually get here + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: decompression failed\n"); + return false; + } + + // Decode weights + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const uint32_t total_planes = (tm.m_ccs_index >= 0) ? 2 : 1; + const uint32_t total_weights = tm.m_grid_width * tm.m_grid_height; + + bool block_used_dct = false; + if (use_dct) + block_used_dct = ((mode_byte & XUASTC_LDR_MODE_BYTE_USE_DCT) != 0); + + if (block_used_dct) + { + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(astc_block_width, astc_block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + if (num_dc_levels == DCT_MEAN_LEVELS1) + syms.m_dc_sym = mean1_bytes.get_bits8(); + else + syms.m_dc_sym = mean0_bits.get_bits4(); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = run_bytes.get_bits8(); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::DCT decode error\n"); + return false; + } + + int sign = sign_bits.get_bits1(); + int coeff = coeff_bytes.get_bits8() + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + if (num_weight_levels < 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = (prev_w + r) & 3; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels < 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = (prev_w + r) & 7; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels < 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels == 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = (prev_w + r) & 15; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight8_bytes.get_bits8(); + + uint32_t w = (prev_w + r) % num_weight_levels; + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + + } // plane_iter + + } // if (block_used_dct) + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + } // bx + + } // by + + assert(!cur_run_len); + + const uint32_t final_sync_marker = raw_bits.get_bits(FINAL_SYNC_MARKER_BITS); + if (final_sync_marker != FINAL_SYNC_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Final sync check failed (1)\n"); + return false; + } + + if (comp_mode_dec.m_pBuf != comp_mode_dec.m_pBuf_end) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image_full_zstd::Final sync check failed (2)\n"); + return false; + } + + return true; + } + + bool xuastc_ldr_decompress_image( + const uint8_t* pComp_data_all, size_t comp_data_size_all, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + if (debug_output) + basisu::debug_printf("\n------------------- astc_ldr_t::decompress_image\n"); + + assert(g_initialized); + + astc_block_width = 0; + astc_block_height = 0; + actual_width = 0; + actual_height = 0; + has_alpha = false; + uses_srgb_astc_decode_mode = false; + + if (!g_initialized) + { + BASISU_DEVEL_ERROR("Not initialized"); + //dec_blocks.clear(); + return false; + } + + if (comp_data_size_all < 1) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + const uint8_t first_comp_byte = pComp_data_all[0]; + + if (first_comp_byte == (uint8_t)xuastc_ldr_syntax::cFullZStd) + { + return xuastc_ldr_decompress_image_full_zstd( + pComp_data_all, comp_data_size_all, + astc_block_width, astc_block_height, + actual_width, actual_height, has_alpha, uses_srgb_astc_decode_mode, + debug_output, + pInit_callback, pInit_callback_data, + pBlock_callback, pBlock_callback_data); + } + + // Either full arith or hybrid arith+zstd now + + const xuastc_ldr_arith_header* pHdr = nullptr; + + const uint8_t* pComp_data = pComp_data_all + 1; + size_t comp_data_size = comp_data_size_all - 1; + + basisu::uint8_vec uncomp_mean0_bits, uncomp_mean1_bytes, uncomp_run_bytes, uncomp_coeff_bytes, uncomp_weight2_bytes, uncomp_weight3_bytes, uncomp_weight4_bytes, uncomp_weight8_bytes; + simplified_bitwise_decoder mean0_bits, mean1_bytes, run_bytes, coeff_bytes, sign_bits, weight2_bits, weight3_bits, weight4_bits, weight8_bytes; + bool use_fast_decoding = false; + + if (first_comp_byte == (uint8_t)xuastc_ldr_syntax::cHybridArithZStd) + { + if (comp_data_size_all < sizeof(xuastc_ldr_arith_header)) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + pHdr = (const xuastc_ldr_arith_header*)pComp_data_all; + + if (pHdr->m_arith_bytes_len < arith::ArithMinExpectedDataBufSize) + { + BASISU_DEVEL_ERROR("Invalid header\n"); + return false; + } + + const uint64_t total_comp_size = (uint64_t)((uint32_t)pHdr->m_arith_bytes_len) + + pHdr->m_mean0_bits_len + pHdr->m_mean1_bytes_len + + pHdr->m_run_bytes_len + pHdr->m_coeff_bytes_len + pHdr->m_sign_bits_len + + pHdr->m_weight2_bits_len + pHdr->m_weight3_bits_len + pHdr->m_weight4_bits_len + pHdr->m_weight8_bytes_len; + + if ((sizeof(xuastc_ldr_arith_header) + total_comp_size) > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + pComp_data = pComp_data_all + sizeof(xuastc_ldr_arith_header); + comp_data_size = pHdr->m_arith_bytes_len; + + const uint8_t* pCur_buf = (const uint8_t*)pComp_data + comp_data_size; + + // mean0 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_mean0_bits_len, uncomp_mean0_bits); + if (!status) + return false; + pCur_buf += pHdr->m_mean0_bits_len; + mean0_bits.init(uncomp_mean0_bits); + } + + // mean1 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_mean1_bytes_len, uncomp_mean1_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_mean1_bytes_len; + mean1_bytes.init(uncomp_mean1_bytes); + } + + // run + { + bool status = zstd_decompress(pCur_buf, pHdr->m_run_bytes_len, uncomp_run_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_run_bytes_len; + run_bytes.init(uncomp_run_bytes); + } + + // coeff + { + bool status = zstd_decompress(pCur_buf, pHdr->m_coeff_bytes_len, uncomp_coeff_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_coeff_bytes_len; + coeff_bytes.init(uncomp_coeff_bytes); + } + + // sign + { + sign_bits.init(pCur_buf, pHdr->m_sign_bits_len); + pCur_buf += pHdr->m_sign_bits_len; + } + + // weight2 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight2_bits_len, uncomp_weight2_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight2_bits_len; + weight2_bits.init(uncomp_weight2_bytes); + } + + // weight3 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight3_bits_len, uncomp_weight3_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight3_bits_len; + weight3_bits.init(uncomp_weight3_bytes); + } + + // weight4 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight4_bits_len, uncomp_weight4_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight4_bits_len; + weight4_bits.init(uncomp_weight4_bytes); + } + + // weight8 + { + bool status = zstd_decompress(pCur_buf, pHdr->m_weight8_bytes_len, uncomp_weight8_bytes); + if (!status) + return false; + pCur_buf += pHdr->m_weight8_bytes_len; + weight8_bytes.init(uncomp_weight8_bytes); + } + + // sanity check + const uint64_t total_read_size = pCur_buf - pComp_data_all; + if (total_read_size > comp_data_size_all) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + use_fast_decoding = true; + } + + if (comp_data_size < arith::ArithMinExpectedDataBufSize) + { + BASISU_DEVEL_ERROR("Compressed file is too small\n"); + return false; + } + + //interval_timer itm; + //itm.start(); + + arith::arith_dec dec; + if (!dec.init(pComp_data, comp_data_size)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid compressed data\n"); + return false; + } + + const uint32_t header_val = dec.get_bits(ARITH_HEADER_MARKER_BITS); + if (header_val != ARITH_HEADER_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Unexpected header marker\n"); + return false; + } + + const uint32_t astc_block_size_index = dec.get_bits(4); + if (astc_block_size_index >= astc_helpers::NUM_ASTC_BLOCK_SIZES) + { + BASISU_DEVEL_ERROR("Invalid block dimension index\n"); + return false; + } + + const uint32_t block_width = astc_helpers::g_astc_block_sizes[astc_block_size_index][0]; + const uint32_t block_height = astc_helpers::g_astc_block_sizes[astc_block_size_index][1]; + + // sanity checks + assert((int)astc_block_size_index == astc_helpers::find_astc_block_size_index(block_width, block_height)); + assert(astc_helpers::is_valid_block_size(block_width, block_height)); + + astc_block_width = block_width; + astc_block_height = block_height; + + //const uint32_t total_block_pixels = block_width * block_height; + + uses_srgb_astc_decode_mode = dec.get_bit(); + + //const astc_helpers::decode_mode dec_mode = uses_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t width = dec.get_bits(16); + const uint32_t height = dec.get_bits(16); + + if ((width < 1) || (height < 1)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid image dimension\n"); + return false; + } + + actual_width = width; + actual_height = height; + + has_alpha = dec.get_bit(); + + const bool use_dct = (dec.get_bits(1) != 0); + + int int_q = 0; + if (use_dct) + int_q = dec.get_bits(8); + + const float dct_q = (float)int_q / 2.0f; + if ((use_dct) && ((dct_q <= 0.0f) || (dct_q > 100.0f))) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid DCT global quality factor\n"); + return false; + } + + if (debug_output) + { + basisu::fmt_debug_printf("astc_ldr_t::decompress_image: block dim: {}x{}, image dim: {}x{}, sRGB decode profile: {}, has_alpha: {}, dct: {} dct_q: {}\n", + block_width, block_height, + width, height, + uses_srgb_astc_decode_mode, has_alpha, + use_dct, dct_q); + } + + const uint32_t num_blocks_x = (width + block_width - 1) / block_width; + const uint32_t num_blocks_y = (height + block_height - 1) / block_height; + + if (pInit_callback) + { + if (!(*pInit_callback)(num_blocks_x, num_blocks_y, block_width, block_height, uses_srgb_astc_decode_mode, dct_q, has_alpha, pInit_callback_data)) + return false; + } + + assert((size_t)astc_block_size_index < std::size(g_encoder_trial_modes)); + const auto& encoder_trial_modes = g_encoder_trial_modes[astc_block_size_index]; + + assert((size_t)astc_block_size_index < std::size(g_grouped_encoder_trial_modes)); + const auto& grouped_encoder_trial_modes = g_grouped_encoder_trial_modes[astc_block_size_index]; + + arith::arith_data_model mode_model((uint32_t)xuastc_mode::cMODE_TOTAL); + + arith::arith_data_model solid_color_dpcm_model[4]; + for (uint32_t i = 0; i < 4; i++) + solid_color_dpcm_model[i].init(256, true); + + arith::arith_data_model raw_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + raw_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + arith::arith_data_model dpcm_endpoint_models[astc_helpers::TOTAL_ENDPOINT_ISE_RANGES]; + for (uint32_t i = 0; i < astc_helpers::TOTAL_ENDPOINT_ISE_RANGES; i++) + dpcm_endpoint_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE + i)); + + arith::arith_bit_model is_base_ofs_model; + arith::arith_bit_model use_dct_model[4]; + arith::arith_bit_model use_dpcm_endpoints_model; + + arith::arith_data_model cem_index_model[8]; + for (uint32_t i = 0; i < 8; i++) + cem_index_model[i].init(OTM_NUM_CEMS); + + arith::arith_data_model subset_index_model[OTM_NUM_SUBSETS]; + for (uint32_t i = 0; i < OTM_NUM_SUBSETS; i++) + subset_index_model[i].init(OTM_NUM_SUBSETS); + + arith::arith_data_model ccs_index_model[OTM_NUM_CCS]; + for (uint32_t i = 0; i < OTM_NUM_CCS; i++) + ccs_index_model[i].init(OTM_NUM_CCS); + + arith::arith_data_model grid_size_model[OTM_NUM_GRID_SIZES]; + for (uint32_t i = 0; i < OTM_NUM_GRID_SIZES; i++) + grid_size_model[i].init(OTM_NUM_GRID_SIZES); + + arith::arith_data_model grid_aniso_model[OTM_NUM_GRID_ANISOS]; + for (uint32_t i = 0; i < OTM_NUM_GRID_ANISOS; i++) + grid_aniso_model[i].init(OTM_NUM_GRID_ANISOS); + + arith::arith_data_model dct_run_len_model; // [0,63] or 64=EOB + arith::arith_data_model dct_coeff_mag; // [1,255] (blocks with larger mags go DPCM) + arith::arith_data_model weight_mean_models[2]; + arith::arith_data_model raw_weight_models[astc_helpers::TOTAL_WEIGHT_ISE_RANGES]; + + if (!use_fast_decoding) + { + // Models used for weight decompression in pure arithmetic mode. + dct_run_len_model.init(65); + dct_coeff_mag.init(255); + + weight_mean_models[0].init(DCT_MEAN_LEVELS0); + weight_mean_models[1].init(DCT_MEAN_LEVELS1); + + for (uint32_t i = 0; i < astc_helpers::TOTAL_WEIGHT_ISE_RANGES; i++) + raw_weight_models[i].init(astc_helpers::get_ise_levels(astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE + i)); + } + + const grid_weight_dct& grid_dct = g_grid_weight_dcts[astc_block_size_index]; + + basisu::vector2D log_blocks; + if (!log_blocks.try_resize(num_blocks_x, 8)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: out of memory\n"); + return false; + } + + memset(log_blocks.get_ptr(), 0, log_blocks.size_in_bytes()); + + basisu::vector2D prev_block_states; + if (!prev_block_states.try_resize(num_blocks_x, 2)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: out of memory\n"); + return false; + } + + arith::arith_data_model submode_models[OTM_NUM_CEMS][OTM_NUM_SUBSETS][OTM_NUM_CCS][OTM_NUM_GRID_SIZES][OTM_NUM_GRID_ANISOS]; + + arith::arith_bit_model endpoints_use_bc_models[4]; + + arith::arith_data_model endpoint_reuse_delta_model(basist::astc_6x6_hdr::NUM_REUSE_XY_DELTAS); + + arith::arith_data_model config_reuse_model[4]; + for (uint32_t i = 0; i < 4; i++) + config_reuse_model[i].init(4); + + arith::arith_gamma_contexts m_run_len_contexts; + uint32_t cur_run_len = 0; + + int part2_hash[PART_HASH_SIZE]; + std::fill(part2_hash, part2_hash + PART_HASH_SIZE, -1); + + int part3_hash[PART_HASH_SIZE]; + std::fill(part3_hash, part3_hash + PART_HASH_SIZE, -1); + + arith::arith_bit_model use_part_hash_model[4]; + arith::arith_data_model part2_hash_index_model(PART_HASH_SIZE, true); + arith::arith_data_model part3_hash_index_model(PART_HASH_SIZE, true); + + //if (debug_output) + // debug_printf("Decompressor init time finish: {} secs\n", itm.get_elapsed_secs()); + + //itm.start(); + + dct_syms syms; + + fvec dct_work; + + for (uint32_t by = 0; by < num_blocks_y; by++) + { + for (uint32_t bx = 0; bx < num_blocks_x; bx++) + { + prev_block_state& new_prev_state = prev_block_states(bx, by & 1); + new_prev_state.clear(); + + const prev_block_state* pLeft_state = bx ? &prev_block_states(bx - 1, by & 1) : nullptr; + const prev_block_state* pUpper_state = by ? &prev_block_states(bx, (by - 1) & 1) : nullptr; + const prev_block_state* pDiag_state = (bx && by) ? &prev_block_states(bx - 1, (by - 1) & 1) : nullptr; + const prev_block_state* pPred_state = pLeft_state ? pLeft_state : pUpper_state; // left or upper, or nullptr on first block + + astc_helpers::log_astc_block& log_blk = log_blocks(bx, by & 7); + + if (cur_run_len) + { + const prev_block_state* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + log_blk = prev_log_blk; + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + new_prev_state.m_used_weight_dct = pPrev_block_state->m_used_weight_dct; + new_prev_state.m_first_endpoint_uses_bc = pPrev_block_state->m_first_endpoint_uses_bc; + new_prev_state.m_reused_full_cfg = true; + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + new_prev_state.m_subset_index = pPrev_block_state->m_subset_index; + new_prev_state.m_ccs_index = pPrev_block_state->m_ccs_index; + new_prev_state.m_grid_size = pPrev_block_state->m_grid_size; + new_prev_state.m_grid_aniso = pPrev_block_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pPrev_block_state->m_used_part_hash; + + cur_run_len--; + continue; + } + + log_blk.clear(); + + uint32_t mode_index = dec.decode_sym(mode_model); + + switch (mode_index) + { + case (uint32_t)xuastc_mode::cMODE_SOLID: + { + const astc_helpers::log_astc_block* pPrev_log_blk = bx ? &log_blocks(bx - 1, by & 7) : (by ? &log_blocks(bx, (by - 1) & 7) : nullptr); + + uint32_t prev_solid_color[4] = { 0 }; + + if (pPrev_log_blk) + { + if (pPrev_log_blk->m_solid_color_flag_ldr) + { + prev_solid_color[0] = pPrev_log_blk->m_solid_color[0] >> 8; + prev_solid_color[1] = pPrev_log_blk->m_solid_color[1] >> 8; + prev_solid_color[2] = pPrev_log_blk->m_solid_color[2] >> 8; + prev_solid_color[3] = pPrev_log_blk->m_solid_color[3] >> 8; + } + else + { + // Decode previous block's first CEM, use the halfway point as the predictor. + color_rgba prev_l, prev_h; + decode_endpoints(pPrev_log_blk->m_color_endpoint_modes[0], pPrev_log_blk->m_endpoints, pPrev_log_blk->m_endpoint_ise_range, prev_l, prev_h); + + prev_solid_color[0] = (prev_l[0] + prev_h[0] + 1) >> 1; + prev_solid_color[1] = (prev_l[1] + prev_h[1] + 1) >> 1; + prev_solid_color[2] = (prev_l[2] + prev_h[2] + 1) >> 1; + prev_solid_color[3] = (prev_l[3] + prev_h[3] + 1) >> 1; + } + } + + uint32_t r = (prev_solid_color[0] + dec.decode_sym(solid_color_dpcm_model[0])) & 0xFF; + uint32_t g = (prev_solid_color[1] + dec.decode_sym(solid_color_dpcm_model[1])) & 0xFF; + uint32_t b = (prev_solid_color[2] + dec.decode_sym(solid_color_dpcm_model[2])) & 0xFF; + + uint32_t a = 255; + if (has_alpha) + a = (prev_solid_color[3] + dec.decode_sym(solid_color_dpcm_model[3])) & 0xFF; + + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | (r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | (g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | (b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | (a << 8)); + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + // Bias the statistics towards using DCT (most common case). + if (use_dct) + new_prev_state.m_used_weight_dct = true; + + new_prev_state.m_first_endpoint_uses_bc = true; + new_prev_state.m_was_solid_color = true; + new_prev_state.m_tm_index = -1; + new_prev_state.m_base_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + new_prev_state.m_subset_index = 0; + new_prev_state.m_ccs_index = 0; + new_prev_state.m_grid_size = 0; + new_prev_state.m_grid_aniso = 0; + new_prev_state.m_reused_full_cfg = false; + new_prev_state.m_used_part_hash = true; // bias to true + + break; + } + case (uint32_t)xuastc_mode::cMODE_RUN: + { + if (!bx && !by) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run command\n"); + return false; + } + + cur_run_len = dec.decode_gamma(m_run_len_contexts); + if (!cur_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run len\n"); + return false; + } + + const uint32_t max_possible_run_len = num_blocks_x - bx; + if (cur_run_len > max_possible_run_len) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid run len\n"); + return false; + } + + const prev_block_state* pPrev_block_state = pLeft_state ? pLeft_state : pUpper_state; + const astc_helpers::log_astc_block& prev_log_blk = bx ? log_blocks(bx - 1, by & 7) : log_blocks(bx, (by - 1) & 7); + + log_blk = prev_log_blk; + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, prev_log_blk, pBlock_callback_data)) + return false; + } + + new_prev_state.m_was_solid_color = pPrev_block_state->m_was_solid_color; + new_prev_state.m_used_weight_dct = pPrev_block_state->m_used_weight_dct; + new_prev_state.m_first_endpoint_uses_bc = pPrev_block_state->m_first_endpoint_uses_bc; + new_prev_state.m_reused_full_cfg = true; + new_prev_state.m_tm_index = pPrev_block_state->m_tm_index; + new_prev_state.m_base_cem_index = pPrev_block_state->m_base_cem_index; + new_prev_state.m_subset_index = pPrev_block_state->m_subset_index; + new_prev_state.m_ccs_index = pPrev_block_state->m_ccs_index; + new_prev_state.m_grid_size = pPrev_block_state->m_grid_size; + new_prev_state.m_grid_aniso = pPrev_block_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pPrev_block_state->m_used_part_hash; + + cur_run_len--; + + break; + } + case (uint32_t)xuastc_mode::cMODE_RAW: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_UP: + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_DIAG: + { + uint32_t tm_index = 0; + uint32_t actual_cem = 0; + + if (mode_index != (uint32_t)xuastc_mode::cMODE_RAW) + { + // Full config+part ID+endpoint reuse from an immediate neighbor + // + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state* pCfg_state = nullptr; + + switch (mode_index) + { + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_LEFT: cfg_dx = -1; pCfg_state = pLeft_state; break; + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_UP: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case (uint32_t)xuastc_mode::cMODE_REUSE_CFG_ENDPOINTS_DIAG: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::xuastc_ldr_decompress_image::Invalid config reuse\n"); + return false; + } + + const astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + for (uint32_t i = 0; i < cfg_log_blk.m_num_partitions; i++) + log_blk.m_color_endpoint_modes[i] = (uint8_t)actual_cem; + log_blk.m_dual_plane = cfg_log_blk.m_dual_plane; + log_blk.m_color_component_selector = cfg_log_blk.m_color_component_selector; + log_blk.m_num_partitions = cfg_log_blk.m_num_partitions; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + log_blk.m_endpoint_ise_range = cfg_log_blk.m_endpoint_ise_range; + log_blk.m_weight_ise_range = cfg_log_blk.m_weight_ise_range; + log_blk.m_grid_width = cfg_log_blk.m_grid_width; + log_blk.m_grid_height = cfg_log_blk.m_grid_height; + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem) * log_blk.m_num_partitions; + memcpy(log_blk.m_endpoints, cfg_log_blk.m_endpoints, total_endpoint_vals); + + new_prev_state.m_tm_index = pCfg_state->m_tm_index; + new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + new_prev_state.m_subset_index = pCfg_state->m_subset_index; + new_prev_state.m_ccs_index = pCfg_state->m_ccs_index; + new_prev_state.m_grid_size = pCfg_state->m_grid_size; + new_prev_state.m_grid_aniso = pCfg_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pCfg_state->m_used_part_hash; + new_prev_state.m_reused_full_cfg = true; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + if (actual_cem_supports_bc) + { + new_prev_state.m_first_endpoint_uses_bc = astc_helpers::used_blue_contraction(actual_cem, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); + assert(new_prev_state.m_first_endpoint_uses_bc == pCfg_state->m_first_endpoint_uses_bc); + } + } + else + { + uint32_t reused_full_cfg_model_index = 0; + if (pLeft_state) + reused_full_cfg_model_index = pLeft_state->m_reused_full_cfg; + else + reused_full_cfg_model_index = 1; + + if (pUpper_state) + reused_full_cfg_model_index |= pUpper_state->m_reused_full_cfg ? 2 : 0; + else + reused_full_cfg_model_index |= 2; + + const uint32_t config_reuse_index = dec.decode_sym(config_reuse_model[reused_full_cfg_model_index]); + + // TODO: Shared with encoder, make global constant + + //if (config_reuse_index < ldr_astc_block_encode_image_output::cMaxConfigReuseNeighbors) + if (config_reuse_index < cMaxConfigReuseNeighbors) + { + // 0 = left, 1 = upper, 2 = left-upper + int cfg_dx = 0, cfg_dy = 0; + const prev_block_state* pCfg_state = nullptr; + + switch (config_reuse_index) + { + case 0: cfg_dx = -1; pCfg_state = pLeft_state; break; + case 1: cfg_dx = 0; cfg_dy = -1; pCfg_state = pUpper_state; break; + case 2: cfg_dx = -1; cfg_dy = -1; pCfg_state = pDiag_state; break; + default: assert(0); break; + } + + if ((((cfg_dx + (int)bx) < 0) || + ((cfg_dy + (int)by) < 0)) || + (!pCfg_state)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + if (pCfg_state->m_tm_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid config reuse\n"); + return false; + } + + astc_helpers::log_astc_block& cfg_log_blk = log_blocks((int)bx + cfg_dx, ((int)by + cfg_dy) & 7); + + tm_index = pCfg_state->m_tm_index; + log_blk.m_partition_id = cfg_log_blk.m_partition_id; + actual_cem = cfg_log_blk.m_color_endpoint_modes[0]; + + new_prev_state.m_tm_index = pCfg_state->m_tm_index; + new_prev_state.m_base_cem_index = pCfg_state->m_base_cem_index; // base cem not including base+ofs, not actual + new_prev_state.m_subset_index = pCfg_state->m_subset_index; + new_prev_state.m_ccs_index = pCfg_state->m_ccs_index; + new_prev_state.m_grid_size = pCfg_state->m_grid_size; + new_prev_state.m_grid_aniso = pCfg_state->m_grid_aniso; + new_prev_state.m_used_part_hash = pCfg_state->m_used_part_hash; + new_prev_state.m_reused_full_cfg = true; + } + else + { + // -------------------- Decode full ASTC config + { + uint32_t prev_cem_index = astc_helpers::CEM_LDR_RGB_DIRECT; + uint32_t prev_subset_index = 0, prev_ccs_index = 0, prev_grid_size = 0, prev_grid_aniso = 0; + + if (pPred_state) + { + prev_cem_index = pPred_state->m_base_cem_index; + prev_subset_index = pPred_state->m_subset_index; + prev_ccs_index = pPred_state->m_ccs_index; + prev_grid_size = pPred_state->m_grid_size; + prev_grid_aniso = pPred_state->m_grid_aniso; + } + + const uint32_t ldrcem_index = cem_to_ldrcem_index(prev_cem_index); + + uint32_t cem_index = dec.decode_sym(cem_index_model[ldrcem_index]); + uint32_t subset_index = dec.decode_sym(subset_index_model[prev_subset_index]); + uint32_t ccs_index = dec.decode_sym(ccs_index_model[prev_ccs_index]); + uint32_t grid_size_index = dec.decode_sym(grid_size_model[prev_grid_size]); + uint32_t grid_aniso_index = dec.decode_sym(grid_aniso_model[prev_grid_aniso]); + + const basisu::uint_vec& modes = get_tm_candidates(grouped_encoder_trial_modes, cem_index, subset_index, ccs_index, grid_size_index, grid_aniso_index); + uint32_t submode_index = 0; + + if (modes.size() > 1) + { + arith::arith_data_model& submode_model = submode_models[cem_index][subset_index][ccs_index][grid_size_index][grid_aniso_index]; + if (!submode_model.get_num_data_syms()) + submode_model.init(modes.size_u32(), true); + + submode_index = dec.decode_sym(submode_model); + } + + if (submode_index >= modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid mode index\n"); + return false; + } + + tm_index = modes[submode_index]; + + new_prev_state.m_tm_index = tm_index; + new_prev_state.m_base_cem_index = cem_index; + new_prev_state.m_subset_index = subset_index; + new_prev_state.m_ccs_index = ccs_index; + new_prev_state.m_grid_size = grid_size_index; + new_prev_state.m_grid_aniso = grid_aniso_index; + new_prev_state.m_reused_full_cfg = false; + } + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + actual_cem = tm.m_cem; + if ((tm.m_cem == astc_helpers::CEM_LDR_RGB_DIRECT) || (tm.m_cem == astc_helpers::CEM_LDR_RGBA_DIRECT)) + { + // Decode is_base_ofs bit + bool is_base_ofs = dec.decode_bit(is_base_ofs_model); + if (is_base_ofs) + { + if (actual_cem == astc_helpers::CEM_LDR_RGB_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET; + else if (actual_cem == astc_helpers::CEM_LDR_RGBA_DIRECT) + actual_cem = astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET; + } + } + + if (tm.m_num_parts > 1) + { + const uint32_t total_unique_indices = get_total_unique_patterns(astc_block_size_index, tm.m_num_parts); + + uint32_t use_part_model_index = 0; + if (pLeft_state) + use_part_model_index = pLeft_state->m_used_part_hash; + else + use_part_model_index = 1; + if (pUpper_state) + use_part_model_index |= pUpper_state->m_used_part_hash ? 2 : 0; + else + use_part_model_index |= 2; + + int* pPart_hash = (tm.m_num_parts == 2) ? part2_hash : part3_hash; + + bool use_part_hash_flag = dec.decode_bit(use_part_hash_model[use_part_model_index]); + + uint32_t unique_pat_index; + if (!use_part_hash_flag) + { + unique_pat_index = dec.decode_truncated_binary(total_unique_indices); + pPart_hash[part_hash_index(unique_pat_index)] = unique_pat_index; + + new_prev_state.m_used_part_hash = false; + } + else + { + uint32_t hash_index = dec.decode_sym((tm.m_num_parts == 2) ? part2_hash_index_model : part3_hash_index_model); + unique_pat_index = pPart_hash[hash_index]; + + if ((int)unique_pat_index < 0) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid hash_index, decompression failed (file corrupt)\n"); + return false; + } + + new_prev_state.m_used_part_hash = true; + } + + if (unique_pat_index >= get_total_unique_patterns(astc_block_size_index, tm.m_num_parts)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid unique_pat_index, decompression failed (file corrupt)\n"); + return false; + } + + log_blk.m_partition_id = unique_pat_index_to_part_seed(astc_block_size_index, tm.m_num_parts, unique_pat_index); + } + else + { + new_prev_state.m_used_part_hash = true; // bias to true + } + + } // if (config_reuse_index < ldr_astc_block_encode_image_output::cMaxConfigReuseNeighbors) + + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const bool actual_cem_supports_bc = astc_helpers::cem_supports_bc(actual_cem); + + const uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(actual_cem); + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + log_blk.m_color_endpoint_modes[part_iter] = (uint8_t)actual_cem; + + log_blk.m_num_partitions = (uint8_t)tm.m_num_parts; + log_blk.m_dual_plane = (tm.m_ccs_index >= 0); + if (log_blk.m_dual_plane) + log_blk.m_color_component_selector = (uint8_t)tm.m_ccs_index; + + log_blk.m_weight_ise_range = (uint8_t)tm.m_weight_ise_range; + log_blk.m_endpoint_ise_range = (uint8_t)tm.m_endpoint_ise_range; + log_blk.m_grid_width = (uint8_t)tm.m_grid_width; + log_blk.m_grid_height = (uint8_t)tm.m_grid_height; + + // --------------------------------- Decode endpoints + const bool used_dpcm_endpoints_flag = dec.decode_bit(use_dpcm_endpoints_model); + + if (!used_dpcm_endpoints_flag) + { + auto& raw_model = raw_endpoint_models[log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + log_blk.m_endpoints[part_iter * total_endpoint_vals + val_iter] = (uint8_t)dec.decode_sym(raw_model); + } // val_iter + + } // part_iter + } + else + { + // Endpoint DPCM + const int num_endpoint_levels = astc_helpers::get_ise_levels(log_blk.m_endpoint_ise_range); + const auto& endpoint_rank_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_rank_to_ISE; + const auto& endpoint_ise_to_rank = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_rank; + + const uint32_t reuse_delta_index = dec.decode_sym(endpoint_reuse_delta_model); + const int reuse_bx = (int)bx + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_x; + const int reuse_by = (int)by + basist::astc_6x6_hdr::g_reuse_xy_deltas[reuse_delta_index].m_y; + + if ((reuse_bx < 0) || (reuse_by < 0) || (reuse_bx >= (int)num_blocks_x) || (reuse_by >= (int)num_blocks_y)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid reuse delta\n"); + return false; + } + + const astc_helpers::log_astc_block* pEndpoint_pred_log_blk = &log_blocks(reuse_bx, reuse_by & 7); + if (pEndpoint_pred_log_blk->m_solid_color_flag_ldr) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid reuse delta\n"); + return false; + } + + uint32_t bc_model_index = 0; + if (pLeft_state) + bc_model_index = pLeft_state->m_first_endpoint_uses_bc; + else + bc_model_index = 1; + + if (pUpper_state) + bc_model_index |= pUpper_state->m_first_endpoint_uses_bc ? 2 : 0; + else + bc_model_index |= 2; + + if (!pEndpoint_pred_log_blk) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Can't use endpoint DPCM here\n"); + return false; + } + + bool endpoints_use_bc[astc_helpers::MAX_PARTITIONS] = { false }; + + if (actual_cem_supports_bc) + { + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + endpoints_use_bc[part_iter] = dec.decode_bit(endpoints_use_bc_models[bc_model_index]); + } + } + + uint8_t predicted_endpoints[astc_helpers::MAX_PARTITIONS][astc_helpers::MAX_CEM_ENDPOINT_VALS] = { }; + + for (uint32_t part_iter = 0; part_iter < log_blk.m_num_partitions; part_iter++) + { + const bool always_repack_flag = false; + bool blue_contraction_clamped_flag = false, base_ofs_clamped_flag = false; + + // Mini-CEM encoder, to cross CEM domains. + bool conv_status = convert_endpoints_across_cems( + pEndpoint_pred_log_blk->m_color_endpoint_modes[0], pEndpoint_pred_log_blk->m_endpoint_ise_range, pEndpoint_pred_log_blk->m_endpoints, + log_blk.m_color_endpoint_modes[0], log_blk.m_endpoint_ise_range, predicted_endpoints[part_iter], + always_repack_flag, + endpoints_use_bc[part_iter], false, + blue_contraction_clamped_flag, base_ofs_clamped_flag); + + if (!conv_status) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Failed predicting endpoints\n"); + return false; + } + } + + auto& dpcm_model = dpcm_endpoint_models[log_blk.m_endpoint_ise_range - astc_helpers::FIRST_VALID_ENDPOINT_ISE_RANGE]; + + for (uint32_t part_iter = 0; part_iter < tm.m_num_parts; part_iter++) + { + for (uint32_t val_iter = 0; val_iter < total_endpoint_vals; val_iter++) + { + const uint32_t endpoint_idx = part_iter * total_endpoint_vals + val_iter; + + int delta = (uint8_t)dec.decode_sym(dpcm_model); + int e_val = basisu::imod(delta + endpoint_ise_to_rank[predicted_endpoints[part_iter][val_iter]], num_endpoint_levels); + + log_blk.m_endpoints[endpoint_idx] = endpoint_rank_to_ise[e_val]; + + } // val_iter + + } // part_iter + + } // if (!used_dpcm_endpoints_flag) + + if (actual_cem_supports_bc) + { + new_prev_state.m_first_endpoint_uses_bc = astc_helpers::used_blue_contraction(actual_cem, log_blk.m_endpoints, log_blk.m_endpoint_ise_range); + } + + } // if (mode_index != cMODE_RAW) + + // ----------------------------------- Decode weights + if (tm_index >= encoder_trial_modes.size()) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image: invalid tm_index, decompression failed (file corrupt)\n"); + return false; + } + + const trial_mode& tm = encoder_trial_modes[tm_index]; + + const uint32_t total_planes = (tm.m_ccs_index >= 0) ? 2 : 1; + const uint32_t total_weights = tm.m_grid_width * tm.m_grid_height; + + uint32_t use_dct_model_index = 0; + if (use_dct) + { + if (pLeft_state) + use_dct_model_index = pLeft_state->m_used_weight_dct; + else + use_dct_model_index = 1; + + if (pUpper_state) + use_dct_model_index |= pUpper_state->m_used_weight_dct ? 2 : 0; + else + use_dct_model_index |= 2; + } + + bool block_used_dct = false; + if (use_dct) + block_used_dct = dec.decode_bit(use_dct_model[use_dct_model_index]); + + if (use_fast_decoding) + { + if (block_used_dct) + { + new_prev_state.m_used_weight_dct = true; + + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + if (num_dc_levels == DCT_MEAN_LEVELS1) + syms.m_dc_sym = mean1_bytes.get_bits8(); + else + syms.m_dc_sym = mean0_bits.get_bits4(); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = run_bytes.get_bits8(); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode error\n"); + return false; + } + + int sign = sign_bits.get_bits1(); + int coeff = coeff_bytes.get_bits8() + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + if (num_weight_levels <= 4) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight2_bits.get_bits2(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels <= 8) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight3_bits.get_bits4(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else if (num_weight_levels <= 16) + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight4_bits.get_bits4(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + else + { + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = weight8_bytes.get_bits8(); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + } + + } // plane_iter + + } // if (block_used_dct) + } + else + { + if (block_used_dct) + { + new_prev_state.m_used_weight_dct = true; + + const astc_block_grid_data* pGrid_data = find_astc_block_grid_data(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height); + + const uint32_t num_dc_levels = grid_weight_dct::get_num_weight_dc_levels(log_blk.m_weight_ise_range); + syms.m_num_dc_levels = num_dc_levels; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + syms.m_coeffs.resize(0); + + syms.m_dc_sym = dec.decode_sym(weight_mean_models[(num_dc_levels == DCT_MEAN_LEVELS1) ? 1 : 0]); + + uint32_t cur_zig_ofs = 1; + + while (cur_zig_ofs < total_weights) + { + uint32_t run_len = dec.decode_sym(dct_run_len_model); + if (run_len == DCT_RUN_LEN_EOB_SYM_INDEX) + break; + + cur_zig_ofs += run_len; + + if (cur_zig_ofs >= total_weights) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode error\n"); + return false; + } + + int sign = dec.get_bit(); + int coeff = dec.decode_sym(dct_coeff_mag) + 1; + + if (sign) + coeff = -coeff; + + syms.m_coeffs.push_back(dct_syms::coeff(basisu::safe_cast_uint16(run_len), basisu::safe_cast_int16(coeff))); + cur_zig_ofs++; + } + + // weight grid IDCT + if (!grid_dct.decode_block_weights(dct_q, plane_iter, log_blk, nullptr, pGrid_data, nullptr, dct_work, &syms)) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::DCT decode failed\n"); + return false; + } + + } // plane_iter + } + else + { + // Weight grid DPCM (no dependency on other blocks, or between planes, for determinism even when IDCT is used) + const uint32_t num_weight_levels = astc_helpers::get_ise_levels(log_blk.m_weight_ise_range); + const auto& weight_rank_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + for (uint32_t plane_iter = 0; plane_iter < total_planes; plane_iter++) + { + int prev_w = num_weight_levels / 2; + + for (uint32_t weight_iter = 0; weight_iter < total_weights; weight_iter++) + { + uint32_t r = dec.decode_sym(raw_weight_models[log_blk.m_weight_ise_range - astc_helpers::FIRST_VALID_WEIGHT_ISE_RANGE]); + + uint32_t w = r; + w = basisu::imod(prev_w + r, num_weight_levels); + + prev_w = w; + + log_blk.m_weights[plane_iter + weight_iter * total_planes] = (uint8_t)weight_rank_to_ise[w]; + + } // weight_iter + + } // plane_iter + } + + } // use_fast_decoding + + if (pBlock_callback) + { + if (!(*pBlock_callback)(bx, by, log_blk, pBlock_callback_data)) + return false; + } + + break; + } + default: + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Invalid mode\n"); + return false; + } + } + + } // bx + + assert(!cur_run_len); + + } // by + + //if (debug_output) + // debug_printf("Decomp time: {3.3}ms\n", itm.get_elapsed_ms()); + + const uint32_t final_sync_marker = dec.get_bits(FINAL_SYNC_MARKER_BITS); + if (final_sync_marker != FINAL_SYNC_MARKER) + { + BASISU_DEVEL_ERROR("astc_ldr_t::decompress_image::Final sync failed\n"); + return false; + } + + if (debug_output) + basisu::debug_printf("astc_ldr_t::decompress_image: Decode sync OK\n"); + + return true; + } + +} // namespace astc_ldr_t + +#endif // #if BASISD_SUPPORT_XUASTC + +#if BASISD_SUPPORT_XUASTC + +namespace bc7u +{ + //------------------------------------------------------------------------------------------------ + // BC7 mode 0-7 decompression. + // Instead of one monster routine to unpack all the BC7 modes, we're lumping the 3 subset, 2 subset, 1 subset, and dual plane modes together into simple shared routines. + + static inline uint32_t bc7_dequant(uint32_t val, uint32_t pbit, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(pbit < 2); assert(val_bits >= 4 && val_bits <= 8); const uint32_t total_bits = val_bits + 1; val = (val << 1) | pbit; val <<= (8 - total_bits); val |= (val >> total_bits); assert(val <= 255); return val; } + static inline uint32_t bc7_dequant(uint32_t val, uint32_t val_bits) { assert(val < (1U << val_bits)); assert(val_bits >= 4 && val_bits <= 8); val <<= (8 - val_bits); val |= (val >> val_bits); assert(val <= 255); return val; } + + static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w < 4); return (l * (64 - basist::g_bc7_weights2[w]) + h * basist::g_bc7_weights2[w] + 32) >> 6; } + static inline uint32_t bc7_interp3(uint32_t l, uint32_t h, uint32_t w) { assert(w < 8); return (l * (64 - basist::g_bc7_weights3[w]) + h * basist::g_bc7_weights3[w] + 32) >> 6; } + static inline uint32_t bc7_interp4(uint32_t l, uint32_t h, uint32_t w) { assert(w < 16); return (l * (64 - basist::g_bc7_weights4[w]) + h * basist::g_bc7_weights4[w] + 32) >> 6; } + static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w, uint32_t bits) + { + assert(l <= 255 && h <= 255); + switch (bits) + { + case 2: return bc7_interp2(l, h, w); + case 3: return bc7_interp3(l, h, w); + case 4: return bc7_interp4(l, h, w); + default: + break; + } + return 0; + } + + inline uint32_t read_bits32(const uint8_t* pBuf, uint32_t& bit_offset, uint32_t codesize) + { + assert(codesize <= 32); + uint32_t bits = 0; + uint32_t total_bits = 0; + + while (total_bits < codesize) + { + uint32_t byte_bit_offset = bit_offset & 7; + uint32_t bits_to_read = basisu::minimum(codesize - total_bits, 8 - byte_bit_offset); + + uint32_t byte_bits = pBuf[bit_offset >> 3] >> byte_bit_offset; + byte_bits &= ((1 << bits_to_read) - 1); + + bits |= (byte_bits << total_bits); + + total_bits += bits_to_read; + bit_offset += bits_to_read; + } + + return bits; + } + + bool unpack_bc7_mode0_2(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + //const uint32_t SUBSETS = 3; + const uint32_t ENDPOINTS = 6; + const uint32_t COMPS = 3; + const uint32_t WEIGHT_BITS = (mode == 0) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 0) ? 4 : 5; + const uint32_t PBITS = (mode == 0) ? 6 : 0; + const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t part = read_bits32(pBuf, bit_offset, (mode == 0) ? 4 : 6); + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); + + uint32_t pbits[6]; + for (uint32_t p = 0; p < PBITS; p++) + pbits[p] = read_bits32(pBuf, bit_offset, 1); + + uint32_t weights[16]; + for (uint32_t i = 0; i < 16; i++) + weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_third_subset_1[part]) || (i == basist::g_bc7_table_anchor_index_third_subset_2[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)((c == 3) ? 255 : (PBITS ? bc7_dequant(endpoints[e][c], pbits[e], ENDPOINT_BITS) : bc7_dequant(endpoints[e][c], ENDPOINT_BITS))); + + color_rgba block_colors[3][8]; + for (uint32_t s = 0; s < 3; s++) + for (uint32_t i = 0; i < WEIGHT_VALS; i++) + { + for (uint32_t c = 0; c < 3; c++) + block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); + block_colors[s][i][3] = 255; + } + + for (uint32_t i = 0; i < 16; i++) + pPixels[i] = block_colors[basist::g_bc7_partition3[part * 16 + i]][weights[i]]; + + return true; + } + + bool unpack_bc7_mode1_3_7(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + //const uint32_t SUBSETS = 2; + const uint32_t ENDPOINTS = 4; + const uint32_t COMPS = (mode == 7) ? 4 : 3; + const uint32_t WEIGHT_BITS = (mode == 1) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 7) ? 5 : ((mode == 1) ? 6 : 7); + const uint32_t PBITS = (mode == 1) ? 2 : 4; + const uint32_t SHARED_PBITS = (mode == 1) ? true : false; + const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t part = read_bits32(pBuf, bit_offset, 6); + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, ENDPOINT_BITS); + + uint32_t pbits[4]; + for (uint32_t p = 0; p < PBITS; p++) + pbits[p] = read_bits32(pBuf, bit_offset, 1); + + uint32_t weights[16]; + for (uint32_t i = 0; i < 16; i++) + weights[i] = read_bits32(pBuf, bit_offset, ((!i) || (i == basist::g_bc7_table_anchor_index_second_subset[part])) ? (WEIGHT_BITS - 1) : WEIGHT_BITS); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)((c == ((mode == 7U) ? 4U : 3U)) ? 255 : bc7_dequant(endpoints[e][c], pbits[SHARED_PBITS ? (e >> 1) : e], ENDPOINT_BITS)); + + color_rgba block_colors[2][8]; + for (uint32_t s = 0; s < 2; s++) + for (uint32_t i = 0; i < WEIGHT_VALS; i++) + { + for (uint32_t c = 0; c < COMPS; c++) + block_colors[s][i][c] = (uint8_t)bc7_interp(endpoints[s * 2 + 0][c], endpoints[s * 2 + 1][c], i, WEIGHT_BITS); + block_colors[s][i][3] = (COMPS == 3) ? 255 : block_colors[s][i][3]; + } + + for (uint32_t i = 0; i < 16; i++) + pPixels[i] = block_colors[basist::g_bc7_partition2[part * 16 + i]][weights[i]]; + + return true; + } + + bool unpack_bc7_mode4_5(uint32_t mode, const void* pBlock_bits, color_rgba* pPixels) + { + const uint32_t ENDPOINTS = 2; + const uint32_t COMPS = 4; + const uint32_t WEIGHT_BITS = 2; + const uint32_t A_WEIGHT_BITS = (mode == 4) ? 3 : 2; + const uint32_t ENDPOINT_BITS = (mode == 4) ? 5 : 7; + const uint32_t A_ENDPOINT_BITS = (mode == 4) ? 6 : 8; + //const uint32_t WEIGHT_VALS = 1 << WEIGHT_BITS; + //const uint32_t A_WEIGHT_VALS = 1 << A_WEIGHT_BITS; + + uint32_t bit_offset = 0; + const uint8_t* pBuf = static_cast(pBlock_bits); + + if (read_bits32(pBuf, bit_offset, mode + 1) != (1U << mode)) return false; + + const uint32_t comp_rot = read_bits32(pBuf, bit_offset, 2); + const uint32_t index_mode = (mode == 4) ? read_bits32(pBuf, bit_offset, 1) : 0; + + color_rgba endpoints[ENDPOINTS]; + for (uint32_t c = 0; c < COMPS; c++) + for (uint32_t e = 0; e < ENDPOINTS; e++) + endpoints[e][c] = (uint8_t)read_bits32(pBuf, bit_offset, (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); + + const uint32_t weight_bits[2] = { index_mode ? A_WEIGHT_BITS : WEIGHT_BITS, index_mode ? WEIGHT_BITS : A_WEIGHT_BITS }; + + uint32_t weights[16], a_weights[16]; + + for (uint32_t i = 0; i < 16; i++) + (index_mode ? a_weights : weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[index_mode] - ((!i) ? 1 : 0)); + + for (uint32_t i = 0; i < 16; i++) + (index_mode ? weights : a_weights)[i] = read_bits32(pBuf, bit_offset, weight_bits[1 - index_mode] - ((!i) ? 1 : 0)); + + assert(bit_offset == 128); + + for (uint32_t e = 0; e < ENDPOINTS; e++) + for (uint32_t c = 0; c < 4; c++) + endpoints[e][c] = (uint8_t)bc7_dequant(endpoints[e][c], (c == 3) ? A_ENDPOINT_BITS : ENDPOINT_BITS); + + color_rgba block_colors[8]; + for (uint32_t i = 0; i < (1U << weight_bits[0]); i++) + for (uint32_t c = 0; c < 3; c++) + block_colors[i][c] = (uint8_t)bc7_interp(endpoints[0][c], endpoints[1][c], i, weight_bits[0]); + + for (uint32_t i = 0; i < (1U << weight_bits[1]); i++) + block_colors[i][3] = (uint8_t)bc7_interp(endpoints[0][3], endpoints[1][3], i, weight_bits[1]); + + for (uint32_t i = 0; i < 16; i++) + { + pPixels[i] = block_colors[weights[i]]; + pPixels[i].a = block_colors[a_weights[i]].a; + if (comp_rot >= 1) + std::swap(pPixels[i].a, pPixels[i].m_comps[comp_rot - 1]); + } + + return true; + } + + struct bc7_mode_6 + { + struct + { + uint64_t m_mode : 7; + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 7; + uint64_t m_a1 : 7; + uint64_t m_p0 : 1; + } m_lo; + + union + { + struct + { + uint64_t m_p1 : 1; + uint64_t m_s00 : 3; + uint64_t m_s10 : 4; + uint64_t m_s20 : 4; + uint64_t m_s30 : 4; + + uint64_t m_s01 : 4; + uint64_t m_s11 : 4; + uint64_t m_s21 : 4; + uint64_t m_s31 : 4; + + uint64_t m_s02 : 4; + uint64_t m_s12 : 4; + uint64_t m_s22 : 4; + uint64_t m_s32 : 4; + + uint64_t m_s03 : 4; + uint64_t m_s13 : 4; + uint64_t m_s23 : 4; + uint64_t m_s33 : 4; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; + + bool unpack_bc7_mode6(const void* pBlock_bits, color_rgba* pPixels) + { + static_assert(sizeof(bc7_mode_6) == 16, "sizeof(bc7_mode_6) == 16"); + + const bc7_mode_6& block = *static_cast(pBlock_bits); + + if (block.m_lo.m_mode != (1 << 6)) + return false; + + const uint32_t r0 = (uint32_t)((block.m_lo.m_r0 << 1) | block.m_lo.m_p0); + const uint32_t g0 = (uint32_t)((block.m_lo.m_g0 << 1) | block.m_lo.m_p0); + const uint32_t b0 = (uint32_t)((block.m_lo.m_b0 << 1) | block.m_lo.m_p0); + const uint32_t a0 = (uint32_t)((block.m_lo.m_a0 << 1) | block.m_lo.m_p0); + const uint32_t r1 = (uint32_t)((block.m_lo.m_r1 << 1) | block.m_hi.m_p1); + const uint32_t g1 = (uint32_t)((block.m_lo.m_g1 << 1) | block.m_hi.m_p1); + const uint32_t b1 = (uint32_t)((block.m_lo.m_b1 << 1) | block.m_hi.m_p1); + const uint32_t a1 = (uint32_t)((block.m_lo.m_a1 << 1) | block.m_hi.m_p1); + + color_rgba vals[16]; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t w = basist::g_bc7_weights4[i]; + const uint32_t iw = 64 - w; + vals[i].set_noclamp_rgba( + (r0 * iw + r1 * w + 32) >> 6, + (g0 * iw + g1 * w + 32) >> 6, + (b0 * iw + b1 * w + 32) >> 6, + (a0 * iw + a1 * w + 32) >> 6); + } + + pPixels[0] = vals[block.m_hi.m_s00]; + pPixels[1] = vals[block.m_hi.m_s10]; + pPixels[2] = vals[block.m_hi.m_s20]; + pPixels[3] = vals[block.m_hi.m_s30]; + + pPixels[4] = vals[block.m_hi.m_s01]; + pPixels[5] = vals[block.m_hi.m_s11]; + pPixels[6] = vals[block.m_hi.m_s21]; + pPixels[7] = vals[block.m_hi.m_s31]; + + pPixels[8] = vals[block.m_hi.m_s02]; + pPixels[9] = vals[block.m_hi.m_s12]; + pPixels[10] = vals[block.m_hi.m_s22]; + pPixels[11] = vals[block.m_hi.m_s32]; + + pPixels[12] = vals[block.m_hi.m_s03]; + pPixels[13] = vals[block.m_hi.m_s13]; + pPixels[14] = vals[block.m_hi.m_s23]; + pPixels[15] = vals[block.m_hi.m_s33]; + + return true; + } + + int determine_bc7_mode(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + for (uint32_t mode = 0; mode <= 7; mode++) + { + if (first_byte & (1U << mode)) + return mode; + } + + return -1; + } + + int determine_bc7_mode_4_index_mode(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + // check for mode 4 + if ((first_byte & 31) != 0b10000) + return -1; + + return (first_byte >> 7); + } + + int determine_bc7_mode_4_or_5_rotation(const void* pBlock) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + if ((first_byte & 31) == 0b10000) + { + // mode 4 + return (first_byte >> 5) & 3; + } + + if ((first_byte & 63) == 0b100000) + { + // mode 5 + return first_byte >> 6; + } + + return -1; + } + + bool unpack_bc7(const void* pBlock, color_rgba* pPixels) + { + const uint32_t first_byte = static_cast(pBlock)[0]; + + for (uint32_t mode = 0; mode <= 7; mode++) + { + if (first_byte & (1U << mode)) + { + switch (mode) + { + case 0: + case 2: + return unpack_bc7_mode0_2(mode, pBlock, pPixels); + case 1: + case 3: + case 7: + return unpack_bc7_mode1_3_7(mode, pBlock, pPixels); + case 4: + case 5: + return unpack_bc7_mode4_5(mode, pBlock, pPixels); + case 6: + return unpack_bc7_mode6(pBlock, pPixels); + default: + break; + } + } + } + + return false; + } +} + +// BASISU_BC7F_USE_SSE41 - only very minimally tested. Needs more testing and more variants. Only improves transcoding perf by ~10% in native so far. +#define BASISU_BC7F_USE_SSE41 (0) +#define BASISU_BC7F_PERF_STATS (0) + +namespace bc7f +{ + const uint32_t MAX_PATTERNS2_TO_CHECK = 64; + const uint32_t MAX_PATTERNS3_TO_CHECK = 64; + + const float UNIQUE_PBIT_DISCOUNT = .85f; + const float SHARED_PBIT_DISCOUNT = .95f; + + //static inline uint8_t mul_8(uint32_t v, uint32_t q) { v = v * q + 128; return (uint8_t)((v + (v >> 8)) >> 8); } + //static inline int mul_8bit(int a, int b) { int t = a * b + 128; return (t + (t >> 8)) >> 8; } + //static inline int lerp_8bit(int a, int b, int s) { assert(a >= 0 && a <= 255); assert(b >= 0 && b <= 255); assert(s >= 0 && s <= 255); return a + mul_8bit(b - a, s); } + + static int popcount32(uint32_t x) + { +#if defined(__EMSCRIPTEN__) || defined(__clang__) || defined(__GNUC__) + return __builtin_popcount(x); +#elif defined(_MSC_VER) + return __popcnt(x); +#else + int count = 0; + while (x) + { + x &= (x - 1); + ++count; + } + return count; +#endif + } + +#if BASISU_BC7F_PERF_STATS + // not thread safe (no need/for dev) + uint32_t g_total_rgb_calls; + uint32_t g_total_rgba_calls; + uint32_t g_total_solid_blocks; + + uint32_t g_total_trivial_mode6_blocks; + + uint32_t g_total_dp_valid_chans_rgb; + uint32_t g_total_dp_valid_chans_a; + uint32_t g_total_high_ortho_energy; + + uint32_t g_total_mode02_evals; + uint32_t g_total_mode02_bailouts; + + uint32_t g_total_mode13_evals; + uint32_t g_total_mode13_bailouts; + + uint32_t g_total_mode45_evals; + uint32_t g_total_mode45_bailouts; + + uint32_t g_total_mode7_evals; + uint32_t g_total_mode7_bailouts; +#endif + + inline int fast_roundf_pos_int(float x) + { + assert(x >= 0.0f); + return (int)(x + 0.5f); + } + + inline int fast_roundf_int(float x) + { + return (x >= 0.0f) ? (int)(x + 0.5f) : (int)(x - 0.5f); + } + + inline int fast_floorf_int(float x) + { + int xi = (int)x; // Truncate towards zero + return ((x < 0.0f) && (x != (float)xi)) ? (xi - 1) : xi; + } + + static inline uint32_t from_7(uint32_t v) + { + assert(v < 128); + return (v << 1) | (v >> 6); + } + + static inline uint32_t from_7(uint32_t v, uint32_t p) + { + assert((v < 128) && (p <= 1)); + return (v << 1) | p; + } + + static inline int to_7(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255) && (pbit >= 0) && (pbit <= 1)); + uint32_t e = (uint32_t(c8) + uint32_t(pbit ^ 1)) >> 1; + return basisu::minimum(127, e); + } + + static inline int to_7(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 127 + 127) / 255; + } + + static inline int to_7(float c, int pbit) + { + assert((c >= 0) && (c <= 255.0f)); + return to_7(fast_roundf_pos_int(c), pbit); + } + + static inline int to_7_clamp(float c, int pbit) + { + return to_7(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + static inline int to_5(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 31 + 127) / 255; + } + + static inline int to_5_clamp(float c) + { + return basisu::clamp(fast_roundf_int(c * (31.0f / 255.0f)), 0, 31); + } + + static inline int to_6(int c8) + { + assert((c8 >= 0) && (c8 <= 255)); + return (c8 * 63 + 127) / 255; + } + + static inline int to_6(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255)); + assert((pbit == 0) || (pbit == 1)); + + int q7 = (c8 * 127 + 127) / 255; + + if ((q7 & 1) != pbit) + { + const int lhs = c8 * 127; + const int rhs = 255 * q7; + + if (lhs >= rhs) + { + q7 = (q7 < 127) ? (q7 + 1) : (q7 - 1); + } + else + { + q7 = (q7 > 0) ? (q7 - 1) : (q7 + 1); + } + } + + return q7 >> 1; + } + + static inline int to_6_clamp(float c, int pbit) + { + return to_6(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + static inline uint32_t from_6(uint32_t v, uint32_t p) + { + assert((v < 64) && (p <= 1)); + v = (v << 1) | p; + v = (v << 1) | (v >> 6); + return v; + } + + static inline uint32_t from_4(uint32_t v, uint32_t p) + { + assert((v < 16) && (p <= 1)); + v = (v << 1) | p; + v = (v << 3) | (v >> 2); + return v; + } + + static inline uint32_t from_5(uint32_t v) + { + assert(v < 32); + v = (v << 3) | (v >> 2); + return v; + } + + static inline uint32_t from_5(uint32_t v, uint32_t p) + { + assert((v < 32) && (p <= 1)); + v = (v << 1) | p; + v = (v << 2) | (v >> 4); + return v; + } + + static inline uint32_t from_6(uint32_t v) + { + assert(v < 64); + v = (v << 2) | (v >> 4); + return v; + } + + static inline int to_5(int c8, int pbit) + { + assert((c8 >= 0) && (c8 <= 255)); + assert((pbit == 0) || (pbit == 1)); + + int q6 = (c8 * 63 + 127) / 255; + + if ((q6 & 1) != pbit) + { + const int lhs = c8 * 63; + const int rhs = 255 * q6; + + if (lhs >= rhs) + { + q6 = (q6 < 63) ? (q6 + 1) : (q6 - 1); + } + else + { + q6 = (q6 > 0) ? (q6 - 1) : (q6 + 1); + } + } + + return q6 >> 1; + } + +#if 0 + static inline int to_5(float c, int pbit) + { + assert((c >= 0.0f) && (c <= 255.0f)); + return to_5((int)fast_roundf_pos_int(c), pbit); + } +#endif + + static inline int to_5_clamp(float c, uint32_t pbit) + { + return to_5(basisu::clamp(fast_roundf_int(c), 0, 255), pbit); + } + + //static inline uint32_t bc7_interp(uint32_t l, uint32_t h, uint32_t w) { assert(w <= 64); return (l * (64 - w) + h * w + 32) >> 6; } + //static inline uint32_t bc7_interp2(uint32_t l, uint32_t h, uint32_t w) { assert(w <= 64); int d = h - l; return (int)l + ((d * (int)w + 32) >> 6); } + //static inline uint32_t bc7_interp3(int l, int d, uint32_t w) { assert(w <= 64); return l + ((d * (int)w + 32) >> 6); } + + static vec4F g_bc7_2bit_ls_tab[4]; + static vec4F g_bc7_3bit_ls_tab[8]; + static vec4F g_bc7_4bit_ls_tab[16]; + static uint16_t g_bc7_part2_bitmasks[64]; + static uint32_t g_part3_bitmasks[64]; + + void init() + { + for (uint32_t i = 0; i < 4; i++) + { + float w = (float)basist::g_bc7_weights2[i] * (1.0f / 64.0f); + g_bc7_2bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 8; i++) + { + float w = (float)basist::g_bc7_weights3[i] * (1.0f / 64.0f); + g_bc7_3bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 16; i++) + { + float w = (float)basist::g_bc7_weights4[i] * (1.0f / 64.0f); + g_bc7_4bit_ls_tab[i].set(w * w, (1.0f - w) * w, (1.0f - w) * (1.0f - w), w); + } + + for (uint32_t i = 0; i < 64; i++) + { + uint16_t y = 0; + + for (uint32_t x = 0; x < 16; x++) + y |= (g_bc7_partition2[i * 16 + x] << x); + + g_bc7_part2_bitmasks[i] = y; + } + + for (uint32_t i = 0; i < 64; i++) + { + const uint8_t* pPat = &g_bc7_partition3[i * 16]; + + for (uint32_t j = 0; j < 16; j++) + { + const uint32_t s = pPat[j]; + + if (s == 0) + g_part3_bitmasks[i] |= (1 << j); + else if (s == 1) + g_part3_bitmasks[i] |= (0x10000 << j); + } + } + } + + void encode_mode0_rgb_block(uint8_t* pBlock, uint32_t part_id, // 3 subsets, 4-bits part ID + uint32_t lr[3], uint32_t lg[3], uint32_t lb[3], // 4 bit endpoints + uint32_t hr[3], uint32_t hg[3], uint32_t hb[3], + uint32_t p[6], + const uint8_t* pWeights) // 3-bit weights + { + assert(part_id < 16); + assert((lr[0] | lr[1] | lr[2] | lg[0] | lg[1] | lg[2] | lb[0] | lb[1] | lb[2]) <= 15); + assert((hr[0] | hr[1] | hr[2] | hg[0] | hg[1] | hg[2] | hb[0] | hb[1] | hb[2]) <= 15); + assert((p[0] | p[1] | p[2] | p[3] | p[4] | p[5]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition3[part_id * 16]; + const uint32_t anchor_index0 = g_bc7_table_anchor_index_third_subset_1[part_id]; + const uint32_t anchor_index1 = g_bc7_table_anchor_index_third_subset_2[part_id]; + + uint32_t weight_inv[3] = { 0, 0, 0 }; + + if (pWeights[0] & 4) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 7; + } + + if (pWeights[anchor_index0] & 4) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 7; + } + + if (pWeights[anchor_index1] & 4) + { + std::swap(lr[2], hr[2]); + std::swap(lg[2], hg[2]); + std::swap(lb[2], hb[2]); + std::swap(p[4], p[5]); + weight_inv[2] = 7; + } + + uint64_t low = 1ULL | ((part_id) << 1) | + ((lr[0]) << 5) | ((hr[0]) << 9) | + ((lr[1]) << 13) | ((hr[1]) << 17) | + ((lr[2]) << 21) | ((hr[2]) << 25) | + (uint64_t(lg[0]) << 29) | (uint64_t(hg[0]) << 33) | + (uint64_t(lg[1]) << 37) | (uint64_t(hg[1]) << 41) | + (uint64_t(lg[2]) << 45) | (uint64_t(hg[2]) << 49) | + (uint64_t(lb[0]) << 53) | (uint64_t(hb[0]) << 57) | + (uint64_t(lb[1]) << 61); + + pBlock[0] = (uint8_t)low; + pBlock[1] = (uint8_t)(low >> 8); + pBlock[2] = (uint8_t)(low >> 16); + pBlock[3] = (uint8_t)(low >> 24); + pBlock[4] = (uint8_t)(low >> 32); + pBlock[5] = (uint8_t)(low >> 40); + pBlock[6] = (uint8_t)(low >> 48); + pBlock[7] = (uint8_t)(low >> 56); + + uint64_t high = (lb[1] >> 3) | ((hb[1]) << 1) | ((lb[2]) << 5) | ((hb[2]) << 9) | + ((p[0]) << 13) | ((p[1]) << 14) | ((p[2]) << 15) | ((p[3]) << 16) | ((p[4]) << 17) | ((p[5]) << 18); + + uint32_t ofs = 19; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 7); + if ((i == 0) || (i == anchor_index0) || (i == anchor_index1)) + { + assert((w & 4) == 0); + } +#endif + + high |= (w << ofs); + ofs += (3 - ((i == 0) || (i == anchor_index0) || (i == anchor_index1))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)high; + pBlock[9] = (uint8_t)(high >> 8); + pBlock[10] = (uint8_t)(high >> 16); + pBlock[11] = (uint8_t)(high >> 24); + pBlock[12] = (uint8_t)(high >> 32); + pBlock[13] = (uint8_t)(high >> 40); + pBlock[14] = (uint8_t)(high >> 48); + pBlock[15] = (uint8_t)(high >> 56); + } + + void encode_mode1_rgb_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], // 6-bit endpoints, 2 shared pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], + uint32_t p0, uint32_t p1, + const uint8_t* pWeights) // 3-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1]) <= 63); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1]) <= 63); + assert((p0 | p1) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 4) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + weight_inv[0] = 7; + } + + if (pWeights[anchor_index] & 4) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + weight_inv[1] = 7; + } + + pBlock[0] = (uint8_t)(0b10 | (part_id << 2)); + + uint64_t x = lr[0] | (hr[0] << (6 * 1)); + x |= (lr[1] << (6 * 2)) | (hr[1] << (6 * 3)); + + x |= (lg[0] << (6 * 4)) | (uint64_t(hg[0]) << (6 * 5)); + x |= (uint64_t(lg[1]) << (6 * 6)) | (uint64_t(hg[1]) << (6 * 7)); + + x |= (uint64_t(lb[0]) << (6 * 8)) | (uint64_t(hb[0]) << (6 * 9)); + x |= (uint64_t(lb[1]) << (6 * 10)); + + // 11*6=66 bits total, write first 64 + + pBlock[1] = (uint8_t)x; + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + + pBlock[5] = (uint8_t)(x >> 32); + pBlock[6] = (uint8_t)(x >> 40); + pBlock[7] = (uint8_t)(x >> 48); + pBlock[8] = (uint8_t)(x >> 56); + + pBlock[9] = (uint8_t)((lb[1] >> 4) | (hb[1] << 2)); + + uint64_t y = p0 | (p1 << 1); + uint32_t ofs = 2; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 7); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 4) == 0); + } +#endif + y |= (w << ofs); + + ofs += (3 - ((i == 0) || (i == anchor_index))); + } + assert(48 == ofs); + + pBlock[10] = (uint8_t)y; + pBlock[11] = (uint8_t)(y >> 8); + pBlock[12] = (uint8_t)(y >> 16); + pBlock[13] = (uint8_t)(y >> 24); + pBlock[14] = (uint8_t)(y >> 32); + pBlock[15] = (uint8_t)(y >> 40); + } + + void encode_mode2_rgb_block(uint8_t* pBlock, uint32_t part_id, // 3 subsets, 6-bits part ID + uint32_t lr[3], uint32_t lg[3], uint32_t lb[3], // 5 bit endpoints, no pbits + uint32_t hr[3], uint32_t hg[3], uint32_t hb[3], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lr[2] | lg[0] | lg[1] | lg[2] | lb[0] | lb[1] | lb[2]) <= 31); + assert((hr[0] | hr[1] | hr[2] | hg[0] | hg[1] | hg[2] | hb[0] | hb[1] | hb[2]) <= 31); + + const uint8_t* pPart_map = &g_bc7_partition3[part_id * 16]; + + uint32_t weight_inv[3] = { 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + weight_inv[0] = 3; + } + + const uint32_t anchor_index0 = g_bc7_table_anchor_index_third_subset_1[part_id]; + if (pWeights[anchor_index0] & 2) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + weight_inv[1] = 3; + } + + const uint32_t anchor_index1 = g_bc7_table_anchor_index_third_subset_2[part_id]; + if (pWeights[anchor_index1] & 2) + { + std::swap(lr[2], hr[2]); + std::swap(lg[2], hg[2]); + std::swap(lb[2], hb[2]); + weight_inv[2] = 3; + } + + uint64_t v = 0b100 | (part_id << 3); + v |= (lr[0] << 9) | (hr[0] << (9 + 5 * 1)); + v |= (lr[1] << (9 + 5 * 2)) | (hr[1] << (9 + 5 * 3)); + v |= (uint64_t(lr[2]) << (9 + 5 * 4)) | (uint64_t(hr[2]) << (9 + 5 * 5)); + + v |= (uint64_t(lg[0]) << (9 + 5 * 6)) | (uint64_t(hg[0]) << (9 + 5 * 7)); + v |= (uint64_t(lg[1]) << (9 + 5 * 8)) | (uint64_t(hg[1]) << (9 + 5 * 9)); + v |= (uint64_t(lg[2]) << (9 + 5 * 10)); + + pBlock[0] = (uint8_t)v; + pBlock[1] = (uint8_t)(v >> 8); + pBlock[2] = (uint8_t)(v >> 16); + pBlock[3] = (uint8_t)(v >> 24); + pBlock[4] = (uint8_t)(v >> 32); + pBlock[5] = (uint8_t)(v >> 40); + pBlock[6] = (uint8_t)(v >> 48); + pBlock[7] = (uint8_t)(v >> 56); + + uint64_t v1 = hg[2]; + v1 |= (lb[0] << (5 * 1)) | (hb[0] << (5 * 2)); + v1 |= (lb[1] << (5 * 3)) | (hb[1] << (5 * 4)); + v1 |= (lb[2] << (5 * 5)) | (uint64_t(hb[2]) << (5 * 6)); + + pBlock[8] = (uint8_t)(v1); + pBlock[9] = (uint8_t)(v1 >> 8); + pBlock[10] = (uint8_t)(v1 >> 16); + pBlock[11] = (uint8_t)(v1 >> 24); + + v1 >>= 32; + + // 3 bits left over + uint32_t ofs = 3; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index0) || (i == anchor_index1)) + { + assert((w & 2) == 0); + } +#endif + v1 |= (w << ofs); + + ofs += (2 - ((i == 0) || (i == anchor_index0) || (i == anchor_index1))); + } + assert(32 == ofs); + + pBlock[12] = (uint8_t)v1; + pBlock[13] = (uint8_t)(v1 >> 8); + pBlock[14] = (uint8_t)(v1 >> 16); + pBlock[15] = (uint8_t)(v1 >> 24); + } + + void encode_mode3_rgb_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], // 7-bit endpoints, 4 unique pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], + uint32_t p[4], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1]) <= 127); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1]) <= 127); + assert((p[0] | p[1] | p[2] | p[3]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); + std::swap(lg[0], hg[0]); + std::swap(lb[0], hb[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 3; + } + + if (pWeights[anchor_index] & 2) + { + std::swap(lr[1], hr[1]); + std::swap(lg[1], hg[1]); + std::swap(lb[1], hb[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 3; + } + + uint64_t x = 0b1000 | (part_id << 4) | + (lr[0] << 10) | (hr[0] << 17) | + (lr[1] << 24) | (uint64_t(hr[1]) << 31) | + (uint64_t(lg[0]) << 38) | (uint64_t(hg[0]) << 45) | + (uint64_t(lg[1]) << 52) | (uint64_t(hg[1]) << 59); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + pBlock[7] = (uint8_t)(x >> 56); + + // 2 bits of hg[1] remaining to pack + + uint64_t y = (hg[1] >> 5) | (lb[0] << 2) | (hb[0] << 9) | + (lb[1] << (9 + 7 * 1)) | (hb[1] << (9 + 7 * 2)) | + (uint64_t(p[0]) << (9 + 7 * 3)) | (uint64_t(p[1]) << (9 + 7 * 3 + 1)) | + (uint64_t(p[2]) << (9 + 7 * 3 + 2)) | (uint64_t(p[3]) << (9 + 7 * 3 + 3)); + + // now 34 total bits + + uint32_t ofs = 34; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 2) == 0); + } +#endif + + y |= (w << ofs); + ofs += (2 - ((i == 0) || (i == anchor_index))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + void encode_mode4_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, // 5-bit RGB endpoints, 6-bit A endpoints, no p-bits + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, + const uint8_t* pWeights0, const uint8_t* pWeights1, // weights0 are 3-bits (RGB), weights1 are 2-bits (alpha) + uint32_t rot_index, uint32_t index_flag) // rot_index=0 no rotation, if index_flag is 1, the 3-bit indices are for RGB + { + assert((lr | lg | lb | hr | hg | hb) <= 31); + assert((la | ha) <= 63); + assert(rot_index <= 3); + assert(index_flag <= 1); + + // defaults: 2nd plane=always alpha, RGB=3-bit indices, A=2-bits (favoring RGB) + //const uint32_t rot_index = 0, index_flag = 1; + + uint32_t weights_inv[2] = { }; + + const uint8_t* p2BitWeights = index_flag ? pWeights1 : pWeights0; + const uint8_t* p3BitWeights = index_flag ? pWeights0 : pWeights1; + + // 3-bits + if (p3BitWeights[0] & 4) + { + weights_inv[0] = 7; + if (index_flag) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + } + else + { + std::swap(la, ha); + } + } + + // 2-bits + if (p2BitWeights[0] & 2) + { + weights_inv[1] = 3; + if (index_flag) + { + std::swap(la, ha); + } + else + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + } + } + + pBlock[0] = (uint8_t)(0b10000 | (rot_index << 5) | (index_flag << 7)); + + // 6*5+6*2=42 bits + uint64_t x = lr | (hr << (5 * 1)); + x |= (lg << (5 * 2)) | (hg << (5 * 3)); + x |= (lb << (5 * 4)) | (hb << (5 * 5)); + x |= (uint64_t(la) << (5 * 6)) | (uint64_t(ha) << (5 * 6 + 6)); + + pBlock[1] = (uint8_t)x; + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + + pBlock[5] = (uint8_t)(x >> 32); + + // 2 leftover bits + x >>= 40; + uint32_t ofs0 = 2; + + // alpha indices (2-bits) + for (uint32_t i = 0; i < 16; i++) + { + assert(p2BitWeights[i] <= 3); + uint64_t w = p2BitWeights[i] ^ weights_inv[1]; + + assert(i || ((w & 2) == 0)); + + x |= (w << ofs0); + + ofs0 += 2 - (i == 0); + } + + // x = 31+2=33 bits + + pBlock[6] = (uint8_t)x; + pBlock[7] = (uint8_t)(x >> 8); + pBlock[8] = (uint8_t)(x >> 16); + pBlock[9] = (uint8_t)(x >> 24); + + x >>= 32; + + // x = 1 bits + uint32_t ofs1 = 1; + + // rgb indices (3-bits) + for (uint32_t i = 0; i < 16; i++) + { + assert(p3BitWeights[i] <= 7); + uint64_t w = p3BitWeights[i] ^ weights_inv[0]; + + assert(i || ((w & 4) == 0)); + + x |= (w << ofs1); + + ofs1 += 3 - (i == 0); + } + + assert(ofs1 == 48); + + // x=48 bits + pBlock[10] = (uint8_t)x; + pBlock[11] = (uint8_t)(x >> 8); + pBlock[12] = (uint8_t)(x >> 16); + pBlock[13] = (uint8_t)(x >> 24); + pBlock[14] = (uint8_t)(x >> 32); + pBlock[15] = (uint8_t)(x >> 40); + } + + // lossless in RGBA + void pack_mode5_solid(uint8_t* pBlock, const color_rgba& c) + { + pBlock[0] = 0b00100000; + + uint32_t lr = basist::g_bc7_mode_5_optimal_endpoints[c[0]].m_lo; + uint32_t hr = basist::g_bc7_mode_5_optimal_endpoints[c[0]].m_hi; + + uint32_t lg = basist::g_bc7_mode_5_optimal_endpoints[c[1]].m_lo; + uint32_t hg = basist::g_bc7_mode_5_optimal_endpoints[c[1]].m_hi; + + uint32_t lb = basist::g_bc7_mode_5_optimal_endpoints[c[2]].m_lo; + uint32_t hb = basist::g_bc7_mode_5_optimal_endpoints[c[2]].m_hi; + + // 8 endpoints are 8-bits, nothing fancy needed + uint32_t a = c[3]; + + // 58 total bits + uint64_t x = lr | (hr << (7 * 1)); + x |= (lg << (7 * 2)) | (hg << (7 * 3)); + x |= (((uint64_t)lb) << (7 * 4)) | (((uint64_t)hb) << (7 * 5)); + x |= (((uint64_t)a) << (7 * 6)) | (((uint64_t)a) << (7 * 6 + 8)); + + // write 56 bits, leaving 2 left over + pBlock[1] = (uint8_t)(x); + pBlock[2] = (uint8_t)(x >> 8); + pBlock[3] = (uint8_t)(x >> 16); + pBlock[4] = (uint8_t)(x >> 24); + pBlock[5] = (uint8_t)(x >> 32); + pBlock[6] = (uint8_t)(x >> 40); + pBlock[7] = (uint8_t)(x >> 48); + + x >>= 56; + assert(x <= 3); + +#if 0 + x |= (0b0101010101010101010101010101011ull << 2); + + pBlock[8] = (uint8_t)(x); + pBlock[9] = (uint8_t)(x >> 8); + pBlock[10] = (uint8_t)(x >> 16); + pBlock[11] = (uint8_t)(x >> 24); + pBlock[12] = (uint8_t)(x >> 32); + pBlock[13] = 0; + pBlock[14] = 0; + pBlock[15] = 0; +#elif 0 + // 0xaaaaaaac | x + pBlock[8] = (uint8_t)(x) | 0xAC; + + static const uint8_t s_tail_bytes[7] = { 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + memcpy(pBlock + 9, s_tail_bytes, 7); +#elif 1 + static const uint8_t s_tail_bytes[8] = { 0xac, 0xaa, 0xaa, 0xaa, 0, 0, 0, 0 }; + memcpy(pBlock + 8, s_tail_bytes, 8); + pBlock[8] |= (uint8_t)x; +#endif + } + + void encode_mode5_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, // 7-bit RGB endpoints, 8-bit alpha endpoints + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, + const uint8_t* pColorWeights, const uint8_t* pAlphaWeights, // both 2-bit weights + uint32_t rot_index = 0) // rot_index=0 no rotation + { + assert((lr | lg | lb | hr | hg | hb) <= 127); + assert((la | ha) <= 255); + assert(rot_index <= 3); + + uint32_t color_inv = 0, alpha_inv = 0; + + if (pColorWeights[0] & 2) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + color_inv = 3; + } + + if (pAlphaWeights[0] & 2) + { + std::swap(la, ha); + alpha_inv = 3; + } + + uint64_t low = (1ULL << 5) | (rot_index << 6) | + (lr << 8) | (hr << 15) | + (lg << 22) | (uint64_t(hg) << 29) | + (uint64_t(lb) << 36) | (uint64_t(hb) << 43) | + (uint64_t(la) << 50) | (uint64_t(ha) << 58); + + pBlock[0] = (uint8_t)low; + pBlock[1] = (uint8_t)(low >> 8); + pBlock[2] = (uint8_t)(low >> 16); + pBlock[3] = (uint8_t)(low >> 24); + pBlock[4] = (uint8_t)(low >> 32); + pBlock[5] = (uint8_t)(low >> 40); + pBlock[6] = (uint8_t)(low >> 48); + pBlock[7] = (uint8_t)(low >> 56); + + uint64_t high = (ha >> 6) & 3; + + uint32_t ofs = 2; + + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pColorWeights[i] ^ color_inv; +#ifdef _DEBUG + assert(w <= 3); + if (i == 0) + { + assert((w & 2) == 0); + } +#endif + high |= (w << ofs); + ofs += (2 - (i == 0)); + } + + assert(33 == ofs); + + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pAlphaWeights[i] ^ alpha_inv; +#ifdef _DEBUG + assert(w <= 3); + if (i == 0) + { + assert((w & 2) == 0); + } +#endif + high |= (w << ofs); + ofs += (2 - (i == 0)); + } + + assert(64 == ofs); + + pBlock[8] = (uint8_t)high; + pBlock[9] = (uint8_t)(high >> 8); + pBlock[10] = (uint8_t)(high >> 16); + pBlock[11] = (uint8_t)(high >> 24); + pBlock[12] = (uint8_t)(high >> 32); + pBlock[13] = (uint8_t)(high >> 40); + pBlock[14] = (uint8_t)(high >> 48); + pBlock[15] = (uint8_t)(high >> 56); + } + + void encode_mode6_rgba_block(uint8_t* pBlock, + uint32_t lr, uint32_t lg, uint32_t lb, uint32_t la, uint32_t p0, // 7-bit endpoints, 2 shared p-bits + uint32_t hr, uint32_t hg, uint32_t hb, uint32_t ha, uint32_t p1, + const uint8_t* pWeights) // 4-bit weights + { + assert((lr | lg | lb | la | hr | hg | hb | ha) <= 127); + assert((p0 | p1) <= 1); + + uint32_t weight_inv = 0; + if (pWeights[0] & 8) + { + std::swap(lr, hr); + std::swap(lg, hg); + std::swap(lb, hb); + std::swap(la, ha); + std::swap(p0, p1); + weight_inv = 15; + } + + // 9*7=63 bits + uint64_t x = 0b1000000 | (lr << (7 * 1)) | (hr << (7 * 2)); + x |= (lg << (7 * 3)) | (uint64_t(hg) << (7 * 4)); + x |= (uint64_t(lb) << (7 * 5)) | (uint64_t(hb) << (7 * 6)); + x |= (uint64_t(la) << (7 * 7)) | (uint64_t(ha) << (7 * 8)); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + x >>= 56; + + // x=7 bits + x |= (p0 << 7); + pBlock[7] = (uint8_t)x; + + uint64_t y = p1; + uint32_t ofs = 1; + // TODO: Unroll/optimize + for (uint32_t i = 0; i < 16; i++) + { + uint64_t w = pWeights[i] ^ weight_inv; + assert(w <= 15); + assert(i || ((w & 8) == 0)); + y |= (w << ofs); + ofs += 3 + (i > 0); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + void encode_mode7_rgba_block(uint8_t* pBlock, uint32_t part_id, // 2 subsets, 6-bits part ID + uint32_t lr[2], uint32_t lg[2], uint32_t lb[2], uint32_t la[2], // 5-bit endpoints, unique pbits + uint32_t hr[2], uint32_t hg[2], uint32_t hb[2], uint32_t ha[2], + uint32_t p[4], + const uint8_t* pWeights) // 2-bit weights + { + assert(part_id < 64); + assert((lr[0] | lr[1] | lg[0] | lg[1] | lb[0] | lb[1] | la[0] | la[1]) <= 31); + assert((hr[0] | hr[1] | hg[0] | hg[1] | hb[0] | hb[1] | ha[0] | ha[1]) <= 31); + assert((p[0] | p[1] | p[2] | p[3]) <= 1); + + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + const uint32_t anchor_index = g_bc7_table_anchor_index_second_subset[part_id]; + + uint32_t weight_inv[2] = { 0, 0 }; + if (pWeights[0] & 2) + { + std::swap(lr[0], hr[0]); std::swap(lg[0], hg[0]); std::swap(lb[0], hb[0]); std::swap(la[0], ha[0]); + std::swap(p[0], p[1]); + weight_inv[0] = 3; + } + + if (pWeights[anchor_index] & 2) + { + std::swap(lr[1], hr[1]); std::swap(lg[1], hg[1]); std::swap(lb[1], hb[1]); std::swap(la[1], ha[1]); + std::swap(p[2], p[3]); + weight_inv[1] = 3; + } + + uint64_t x = 0x80ULL | (part_id << 8) | + (lr[0] << 14) | (hr[0] << 19) | (lr[1] << 24) | (uint64_t(hr[1]) << 29) | + (uint64_t(lg[0]) << 34) | (uint64_t(hg[0]) << 39) | (uint64_t(lg[1]) << 44) | (uint64_t(hg[1]) << 49) | + (uint64_t(lb[0]) << 54) | (uint64_t(hb[0]) << 59); + + pBlock[0] = (uint8_t)x; + pBlock[1] = (uint8_t)(x >> 8); + pBlock[2] = (uint8_t)(x >> 16); + pBlock[3] = (uint8_t)(x >> 24); + + pBlock[4] = (uint8_t)(x >> 32); + pBlock[5] = (uint8_t)(x >> 40); + pBlock[6] = (uint8_t)(x >> 48); + pBlock[7] = (uint8_t)(x >> 56); + + uint64_t y = (lb[1] << 0) | (hb[1] << 5) | + (la[0] << 10) | (ha[0] << 15) | (la[1] << 20) | (ha[1] << 25) | + (uint64_t(p[0]) << 30) | (uint64_t(p[1]) << 31) | (uint64_t(p[2]) << 32) | (uint64_t(p[3]) << 33); + + uint32_t ofs = 34; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + uint64_t w = pWeights[i] ^ weight_inv[subset_index]; + +#ifdef _DEBUG + assert(w <= 3); + if ((i == 0) || (i == anchor_index)) + { + assert((w & 2) == 0); + } +#endif + + y |= (w << ofs); + ofs += (2 - ((i == 0) || (i == anchor_index))); + } + assert(64 == ofs); + + pBlock[8] = (uint8_t)y; + pBlock[9] = (uint8_t)(y >> 8); + pBlock[10] = (uint8_t)(y >> 16); + pBlock[11] = (uint8_t)(y >> 24); + + pBlock[12] = (uint8_t)(y >> 32); + pBlock[13] = (uint8_t)(y >> 40); + pBlock[14] = (uint8_t)(y >> 48); + pBlock[15] = (uint8_t)(y >> 56); + } + + static bool compute_least_squares_endpoints_1D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + float& xl, float& xh, + const color_rgba* pColors, uint32_t comp_index, + float t_r) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][comp_index]; + } + + float q10_r = t_r - q00_r; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + return true; + } + + static bool compute_least_squares_endpoints_3D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + vec4F& xl, vec4F& xh, + const color_rgba* pColors, + float t_r, float t_g, float t_b) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q00_g = 0.0f, q00_b = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][0]; + q00_g += w * (float)pColors[i][1]; + q00_b += w * (float)pColors[i][2]; + } + + float q10_r = t_r - q00_r; + float q10_g = t_g - q00_g; + float q10_b = t_b - q00_b; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh[0] = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl[0] = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + xh[1] = basisu::clamp(iz00 * q00_g + iz01 * q10_g, 0.0f, 255.0f); + xl[1] = basisu::clamp(iz10 * q00_g + iz11 * q10_g, 0.0f, 255.0f); + + xh[2] = basisu::clamp(iz00 * q00_b + iz01 * q10_b, 0.0f, 255.0f); + xl[2] = basisu::clamp(iz10 * q00_b + iz11 * q10_b, 0.0f, 255.0f); + + xh[3] = 0; + xl[3] = 0; + + return true; + } + + static bool compute_least_squares_endpoints_4D( + uint32_t N, const uint8_t* pWeights, uint32_t num_weights, + const vec4F* pSelector_weights, + vec4F& xl, vec4F& xh, + const color_rgba* pColors, + float t_r, float t_g, float t_b, float t_a) + { + BASISU_NOTE_UNUSED(num_weights); + + float z00 = 0.0f, z10 = 0.0f, z11 = 0.0f; + float q00_r = 0.0f, q00_g = 0.0f, q00_b = 0.0f, q00_a = 0.0f; + + for (uint32_t i = 0; i < N; i++) + { + const uint32_t sel = pWeights[i]; + assert(sel < num_weights); + + z00 += pSelector_weights[sel][0]; + z10 += pSelector_weights[sel][1]; + z11 += pSelector_weights[sel][2]; + + const float w = pSelector_weights[sel][3]; + + q00_r += w * (float)pColors[i][0]; + q00_g += w * (float)pColors[i][1]; + q00_b += w * (float)pColors[i][2]; + q00_a += w * (float)pColors[i][3]; + } + + float q10_r = t_r - q00_r; + float q10_g = t_g - q00_g; + float q10_b = t_b - q00_b; + float q10_a = t_a - q00_a; + + float z01 = z10; + + float det = z00 * z11 - z01 * z10; + if (fabs(det) < 1e-8f) + return false; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11 * det; + iz01 = -z01 * det; + iz10 = -z10 * det; + iz11 = z00 * det; + + xh[0] = basisu::clamp(iz00 * q00_r + iz01 * q10_r, 0.0f, 255.0f); + xl[0] = basisu::clamp(iz10 * q00_r + iz11 * q10_r, 0.0f, 255.0f); + + xh[1] = basisu::clamp(iz00 * q00_g + iz01 * q10_g, 0.0f, 255.0f); + xl[1] = basisu::clamp(iz10 * q00_g + iz11 * q10_g, 0.0f, 255.0f); + + xh[2] = basisu::clamp(iz00 * q00_b + iz01 * q10_b, 0.0f, 255.0f); + xl[2] = basisu::clamp(iz10 * q00_b + iz11 * q10_b, 0.0f, 255.0f); + + xh[3] = basisu::clamp(iz00 * q00_a + iz01 * q10_a, 0.0f, 255.0f); + xl[3] = basisu::clamp(iz10 * q00_a + iz11 * q10_a, 0.0f, 255.0f); + + return true; + } + +#if BASISU_BC7F_USE_SSE41 + void bc7_proj_minmax_indices_sse41(const color_rgba* __restrict pPixels, int saxis_r, int saxis_g, int saxis_b, int* out_min_idx, int* out_max_idx) + { + __m128i coef32 = _mm_setr_epi32(saxis_r, saxis_g, saxis_b, 0); // 32-bit lanes + coef32 = _mm_srai_epi32(coef32, 4); // arithmetic >>4 in 32-bit + __m128i COEF = _mm_packs_epi32(coef32, coef32); + + const __m128i ZERO = _mm_setzero_si128(); + + __m128i vmin, vmax; + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[0]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(3, 2, 1, 0)); + + vmin = keyed; + vmax = keyed; + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[4]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(7, 6, 5, 4)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[8]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(11, 10, 9, 8)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[12]); + const __m128i lo16 = _mm_unpacklo_epi8(px, ZERO); // [r0 g0 b0 a0 r1 g1 b1 a1] + const __m128i hi16 = _mm_unpackhi_epi8(px, ZERO); // [r2 g2 b2 a2 r3 g3 b3 a3] + + const __m128i lo32p = _mm_madd_epi16(lo16, COEF); + const __m128i hi32p = _mm_madd_epi16(hi16, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + + const __m128i p32p = _mm_unpacklo_epi64(pair01, pair23); + + const __m128i p32 = _mm_slli_epi32(p32p, 4); + + const __m128i keyed = _mm_add_epi32(p32, _mm_set_epi32(15, 14, 13, 12)); + + vmin = _mm_min_epi32(vmin, keyed); + vmax = _mm_max_epi32(vmax, keyed); + } + + __m128i t = _mm_shuffle_epi32(vmin, _MM_SHUFFLE(2, 3, 0, 1)); + vmin = _mm_min_epi32(vmin, t); + t = _mm_shuffle_epi32(vmin, _MM_SHUFFLE(1, 0, 3, 2)); + vmin = _mm_min_epi32(vmin, t); + const int min_keyed = _mm_cvtsi128_si32(vmin); + + t = _mm_shuffle_epi32(vmax, _MM_SHUFFLE(2, 3, 0, 1)); + vmax = _mm_max_epi32(vmax, t); + t = _mm_shuffle_epi32(vmax, _MM_SHUFFLE(1, 0, 3, 2)); + vmax = _mm_max_epi32(vmax, t); + const int max_keyed = _mm_cvtsi128_si32(vmax); + + *out_min_idx = (min_keyed & 0xF); + *out_max_idx = (max_keyed & 0xF); + } + + void eval_weights_mode6_rgb_sse41( + const color_rgba* __restrict pPixels, uint8_t* __restrict pWeights, + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + const int dr = hr - lr; + const int dg = hg - lg; + const int db = hb - lb; + + const float denom = (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db)) + 0.00000125f; + const float f = 15.0f / denom; + + const __m128i ZEROi = _mm_setzero_si128(); + const __m128i FIFTEEN = _mm_set1_epi32(15); + const __m128 F = _mm_set1_ps(f); + const __m128 HALF = _mm_set1_ps(0.5f); + + const __m128i EP16 = _mm_setr_epi16((short)lr, (short)lg, (short)lb, 0, + (short)lr, (short)lg, (short)lb, 0); + + const __m128i COEF = _mm_setr_epi16((short)dr, (short)dg, (short)db, 0, + (short)dr, (short)dg, (short)db, 0); + + for (int i = 0; i < 16; i += 4) + { + const __m128i px = _mm_loadu_si128((const __m128i*) & pPixels[i]); + + const __m128i lo16 = _mm_unpacklo_epi8(px, ZEROi); + const __m128i hi16 = _mm_unpackhi_epi8(px, ZEROi); + + const __m128i lo_adj = _mm_sub_epi16(lo16, EP16); + const __m128i hi_adj = _mm_sub_epi16(hi16, EP16); + + const __m128i lo32p = _mm_madd_epi16(lo_adj, COEF); + const __m128i hi32p = _mm_madd_epi16(hi_adj, COEF); + + const __m128i lo_sum = _mm_add_epi32(lo32p, _mm_shuffle_epi32(lo32p, _MM_SHUFFLE(2, 3, 0, 1))); + const __m128i hi_sum = _mm_add_epi32(hi32p, _mm_shuffle_epi32(hi32p, _MM_SHUFFLE(2, 3, 0, 1))); + + const __m128i pair01 = _mm_shuffle_epi32(lo_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i pair23 = _mm_shuffle_epi32(hi_sum, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i dot32 = _mm_unpacklo_epi64(pair01, pair23); + + __m128 y = _mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(dot32), F), HALF); + __m128i sel32 = _mm_cvttps_epi32(y); + + sel32 = _mm_min_epi32(_mm_max_epi32(sel32, ZEROi), FIFTEEN); + + __m128i sel16 = _mm_packs_epi32(sel32, ZEROi); + __m128i sel8 = _mm_packus_epi16(sel16, ZEROi); + *(uint32_t*)&pWeights[i] = (uint32_t)_mm_cvtsi128_si32(sel8); + } + } +#endif + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, + int lr, + int dr, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + return (re * re); + } + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, int pg, int pb, + int lr, int lg, int lb, + int dr, int dg, int db, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + int ge = pg - (lg + ((dg * (int)w + 32) >> 6)); + int be = pb - (lb + ((db * (int)w + 32) >> 6)); + return (re * re) + (ge * ge) + (be * be); + } + + BASISU_FORCE_INLINE uint32_t bc7_sse( + int pr, int pg, int pb, int pa, + int lr, int lg, int lb, int la, + int dr, int dg, int db, int da, + int w) + { + assert((w >= 0) && (w <= 64)); + int re = pr - (lr + ((dr * (int)w + 32) >> 6)); + int ge = pg - (lg + ((dg * (int)w + 32) >> 6)); + int be = pb - (lb + ((db * (int)w + 32) >> 6)); + int ae = pa - (la + ((da * (int)w + 32) >> 6)); + return (re * re) + (ge * ge) + (be * be) + (ae * ae); + } + + void eval_weights_mode6_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode6_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, + int hr, int hg, int hb, + uint32_t p0, uint32_t p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); + + // assumes packed a's are always 127 + const int la = from_7(127, p0); + const int ha = from_7(127, p1); + const int da = ha - la; + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], pPixels[i + 0][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], pPixels[i + 1][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], pPixels[i + 2][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], pPixels[i + 3][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel3]); + } + + return sse; + } + + void eval_weights_mode6_rgba(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, int la, int p0, + int hr, int hg, int hb, int ha, int p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); la = from_7(la, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); ha = from_7(ha, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + int da = ha - la; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + basisu::squarei(da) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db + la * da); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + pPixels[i + 0][3] * da + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + pPixels[i + 1][3] * da + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + pPixels[i + 2][3] * da + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + pPixels[i + 3][3] * da + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode6_rgba_sse(const color_rgba* pPixels, uint8_t* pWeights, // 4-bits + int lr, int lg, int lb, int la, int p0, + int hr, int hg, int hb, int ha, int p1) + { + lr = from_7(lr, p0); lg = from_7(lg, p0); lb = from_7(lb, p0); la = from_7(la, p0); + hr = from_7(hr, p1); hg = from_7(hg, p1); hb = from_7(hb, p1); ha = from_7(ha, p1); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + int da = ha - la; + + const float f = 15.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + basisu::squarei(da) + .00000125f); + + const int sofs = -(lr * dr + lg * dg + lb * db + la * da); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)(float(pPixels[i + 0][0] * dr + pPixels[i + 0][1] * dg + pPixels[i + 0][2] * db + pPixels[i + 0][3] * da + sofs) * f + .5f); + int sel1 = (int)(float(pPixels[i + 1][0] * dr + pPixels[i + 1][1] * dg + pPixels[i + 1][2] * db + pPixels[i + 1][3] * da + sofs) * f + .5f); + int sel2 = (int)(float(pPixels[i + 2][0] * dr + pPixels[i + 2][1] * dg + pPixels[i + 2][2] * db + pPixels[i + 2][3] * da + sofs) * f + .5f); + int sel3 = (int)(float(pPixels[i + 3][0] * dr + pPixels[i + 3][1] * dg + pPixels[i + 3][2] * db + pPixels[i + 3][3] * da + sofs) * f + .5f); + + if ((uint32_t)sel0 > 15) sel0 = (~sel0 >> 31) & 15; + if ((uint32_t)sel1 > 15) sel1 = (~sel1 >> 31) & 15; + if ((uint32_t)sel2 > 15) sel2 = (~sel2 >> 31) & 15; + if ((uint32_t)sel3 > 15) sel3 = (~sel3 >> 31) & 15; + + pWeights[i + 0] = (uint8_t)sel0; + pWeights[i + 1] = (uint8_t)sel1; + pWeights[i + 2] = (uint8_t)sel2; + pWeights[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], pPixels[i + 0][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], pPixels[i + 1][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], pPixels[i + 2][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], pPixels[i + 3][3], lr, lg, lb, la, dr, dg, db, da, basist::g_bc7_weights4[sel3]); + } + + return sse; + } + + void eval_weights_mode1_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[2], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_6(blr[s], pbits[s]); + lg[s] = from_6(blg[s], pbits[s]); + lb[s] = from_6(blb[s], pbits[s]); + + hr[s] = from_6(bhr[s], pbits[s]); + hg[s] = from_6(bhg[s], pbits[s]); + hb[s] = from_6(bhb[s], pbits[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode1_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[2], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_6(blr[s], pbits[s]); + lg[s] = from_6(blg[s], pbits[s]); + lb[s] = from_6(blb[s], pbits[s]); + + hr[s] = from_6(bhr[s], pbits[s]); + hg[s] = from_6(bhg[s], pbits[s]); + hb[s] = from_6(bhb[s], pbits[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], lr[subset_index], lg[subset_index], lb[subset_index], dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights3[sel]); + } + + return sse; + } + + void eval_weights_mode7_rgba(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bla[2], + uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], uint32_t bha[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], la[2]; + int hr[2], hg[2], hb[2], ha[2]; + int dr[2], dg[2], db[2], da[2]; + + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t l_pbit = pbits[s * 2 + 0], h_pbit = pbits[s * 2 + 1]; + + lr[s] = from_5(blr[s], l_pbit); + lg[s] = from_5(blg[s], l_pbit); + lb[s] = from_5(blb[s], l_pbit); + la[s] = from_5(bla[s], l_pbit); + + hr[s] = from_5(bhr[s], h_pbit); + hg[s] = from_5(bhg[s], h_pbit); + hb[s] = from_5(bhb[s], h_pbit); + ha[s] = from_5(bha[s], h_pbit); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + da[s] = ha[s] - la[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + basisu::squarei(da[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + basisu::squarei(da[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0] + la[0] * da[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] + la[1] * da[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] + ((int)pPixels[i][3]) * da[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode7_rgba_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bla[2], + uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], uint32_t bha[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], la[2]; + int hr[2], hg[2], hb[2], ha[2]; + int dr[2], dg[2], db[2], da[2]; + + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t l_pbit = pbits[s * 2 + 0], h_pbit = pbits[s * 2 + 1]; + + lr[s] = from_5(blr[s], l_pbit); + lg[s] = from_5(blg[s], l_pbit); + lb[s] = from_5(blb[s], l_pbit); + la[s] = from_5(bla[s], l_pbit); + + hr[s] = from_5(bhr[s], h_pbit); + hg[s] = from_5(bhg[s], h_pbit); + hb[s] = from_5(bhb[s], h_pbit); + ha[s] = from_5(bha[s], h_pbit); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + da[s] = ha[s] - la[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + basisu::squarei(da[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + basisu::squarei(da[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0] + la[0] * da[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] + la[1] * da[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] + ((int)pPixels[i][3]) * da[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], pPixels[i][3], + lr[subset_index], lg[subset_index], lb[subset_index], la[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], da[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode3_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_7(blr[s], pbits[s * 2 + 0]); + lg[s] = from_7(blg[s], pbits[s * 2 + 0]); + lb[s] = from_7(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_7(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_7(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_7(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode3_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[2], uint32_t blg[2], uint32_t blb[2], uint32_t bhr[2], uint32_t bhg[2], uint32_t bhb[2], + uint32_t pbits[4], uint32_t subset_bitmask) + { + int lr[2], lg[2], lb[2], hr[2], hg[2], hb[2], dr[2], dg[2], db[2]; + + for (uint32_t s = 0; s < 2; s++) + { + lr[s] = from_7(blr[s], pbits[s * 2 + 0]); + lg[s] = from_7(blg[s], pbits[s * 2 + 0]); + lb[s] = from_7(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_7(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_7(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_7(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[2] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f) + }; + + const int sofs[2] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1] }; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = (subset_bitmask >> i) & 1; + + int sel = (int)((float)( + ((int)pPixels[i][0]) * dr[subset_index] + ((int)pPixels[i][1]) * dg[subset_index] + ((int)pPixels[i][2]) * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], lr[subset_index], lg[subset_index], lb[subset_index], dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode0_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], + uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pbits[6], + uint32_t pat_index) + { + assert(pat_index <= 15); + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_4(blr[s], pbits[s * 2 + 0]); + lg[s] = from_4(blg[s], pbits[s * 2 + 0]); + lb[s] = from_4(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_4(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_4(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_4(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode0_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 3-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], + uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pbits[6], + uint32_t pat_index) + { + assert(pat_index <= 15); + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_4(blr[s], pbits[s * 2 + 0]); + lg[s] = from_4(blg[s], pbits[s * 2 + 0]); + lb[s] = from_4(blb[s], pbits[s * 2 + 0]); + + hr[s] = from_4(bhr[s], pbits[s * 2 + 1]); + hg[s] = from_4(bhg[s], pbits[s * 2 + 1]); + hb[s] = from_4(bhb[s], pbits[s * 2 + 1]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 7.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 7.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 7) + sel = (~sel >> 31) & 7; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], + lr[subset_index], lg[subset_index], lb[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights3[sel]); + } + + return sse; + } + + void eval_weights_mode2_rgb(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pat_index) + { + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_5(blr[s]); + lg[s] = from_5(blg[s]); + lb[s] = from_5(blb[s]); + + hr[s] = from_5(bhr[s]); + hg[s] = from_5(bhg[s]); + hb[s] = from_5(bhb[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + } + } + + uint32_t eval_weights_mode2_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights, // 2-bits + uint32_t blr[3], uint32_t blg[3], uint32_t blb[3], uint32_t bhr[3], uint32_t bhg[3], uint32_t bhb[3], + uint32_t pat_index) + { + int lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], dr[3], dg[3], db[3]; + + for (uint32_t s = 0; s < 3; s++) + { + lr[s] = from_5(blr[s]); + lg[s] = from_5(blg[s]); + lb[s] = from_5(blb[s]); + + hr[s] = from_5(bhr[s]); + hg[s] = from_5(bhg[s]); + hb[s] = from_5(bhb[s]); + + dr[s] = hr[s] - lr[s]; + dg[s] = hg[s] - lg[s]; + db[s] = hb[s] - lb[s]; + } + + const float f[3] = + { + 3.0f / (float)(basisu::squarei(dr[0]) + basisu::squarei(dg[0]) + basisu::squarei(db[0]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[1]) + basisu::squarei(dg[1]) + basisu::squarei(db[1]) + .00000125f), + 3.0f / (float)(basisu::squarei(dr[2]) + basisu::squarei(dg[2]) + basisu::squarei(db[2]) + .00000125f) + }; + + const int sofs[3] = { + lr[0] * dr[0] + lg[0] * dg[0] + lb[0] * db[0], + lr[1] * dr[1] + lg[1] * dg[1] + lb[1] * db[1], + lr[2] * dr[2] + lg[2] * dg[2] + lb[2] * db[2] }; + + const uint8_t* pPart_map = &g_bc7_partition3[pat_index * 16]; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = pPart_map[i]; + + int sel = (int)((float)( + (int)pPixels[i][0] * dr[subset_index] + (int)pPixels[i][1] * dg[subset_index] + (int)pPixels[i][2] * db[subset_index] - sofs[subset_index]) * f[subset_index] + .5f); + + if ((uint32_t)sel > 3) + sel = (~sel >> 31) & 3; + + pWeights[i] = (uint8_t)sel; + + sse += bc7_sse(pPixels[i][0], pPixels[i][1], pPixels[i][2], + lr[subset_index], lg[subset_index], lb[subset_index], + dr[subset_index], dg[subset_index], db[subset_index], basist::g_bc7_weights2[sel]); + } + + return sse; + } + + void eval_weights_mode4_3bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 3-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 7.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_3bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 3-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 7.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights3[sel3]); + } + + return sse; + } + + void eval_weights_mode4_2bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_2bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_5(lr); lg = from_5(lg); lb = from_5(lb); + hr = from_5(hr); hg = from_5(hg); hb = from_5(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode4_2bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_2bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode4_3bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 3-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 7.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode4_3bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 3-bits + int la, int ha) + { + la = from_6(la); + ha = from_6(ha); + + int da = ha - la; + + const float f = 7.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 7) sel0 = (~sel0 >> 31) & 7; + if ((uint32_t)sel1 > 7) sel1 = (~sel1 >> 31) & 7; + if ((uint32_t)sel2 > 7) sel2 = (~sel2 >> 31) & 7; + if ((uint32_t)sel3 > 7) sel3 = (~sel3 >> 31) & 7; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights3[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights3[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights3[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights3[sel3]); + } + + return sse; + } + + void eval_weights_mode5_2bit_rgb(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_7(lr); lg = from_7(lg); lb = from_7(lb); + hr = from_7(hr); hg = from_7(hg); hb = from_7(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode5_2bit_rgb_sse(const color_rgba* pPixels, uint8_t* pWeights0, // 2-bits + int lr, int lg, int lb, + int hr, int hg, int hb) + { + lr = from_7(lr); lg = from_7(lg); lb = from_7(lb); + hr = from_7(hr); hg = from_7(hg); hb = from_7(hb); + + int dr = hr - lr; + int dg = hg - lg; + int db = hb - lb; + + const float f = 3.0f / (float)(basisu::squarei(dr) + basisu::squarei(dg) + basisu::squarei(db) + .00000125f); + + const int sofs = lr * dr + lg * dg + lb * db; + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)((int)pPixels[i + 0][0] * dr + (int)pPixels[i + 0][1] * dg + (int)pPixels[i + 0][2] * db - sofs) * f + .5f); + int sel1 = (int)((float)((int)pPixels[i + 1][0] * dr + (int)pPixels[i + 1][1] * dg + (int)pPixels[i + 1][2] * db - sofs) * f + .5f); + int sel2 = (int)((float)((int)pPixels[i + 2][0] * dr + (int)pPixels[i + 2][1] * dg + (int)pPixels[i + 2][2] * db - sofs) * f + .5f); + int sel3 = (int)((float)((int)pPixels[i + 3][0] * dr + (int)pPixels[i + 3][1] * dg + (int)pPixels[i + 3][2] * db - sofs) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights0[i + 0] = (uint8_t)sel0; + pWeights0[i + 1] = (uint8_t)sel1; + pWeights0[i + 2] = (uint8_t)sel2; + pWeights0[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][0], pPixels[i + 0][1], pPixels[i + 0][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][0], pPixels[i + 1][1], pPixels[i + 1][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][0], pPixels[i + 2][1], pPixels[i + 2][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][0], pPixels[i + 3][1], pPixels[i + 3][2], lr, lg, lb, dr, dg, db, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + void eval_weights_mode5_2bit_a(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + } + } + + uint32_t eval_weights_mode5_2bit_a_sse(const color_rgba* pPixels, uint8_t* pWeights1, // 2-bits + int la, int ha) + { + int da = ha - la; + + const float f = 3.0f / (float)(da + .00000125f); + + uint32_t sse = 0; + + for (uint32_t i = 0; i < 16; i += 4) + { + int sel0 = (int)((float)(pPixels[i + 0][3] - la) * f + .5f); + int sel1 = (int)((float)(pPixels[i + 1][3] - la) * f + .5f); + int sel2 = (int)((float)(pPixels[i + 2][3] - la) * f + .5f); + int sel3 = (int)((float)(pPixels[i + 3][3] - la) * f + .5f); + + if ((uint32_t)sel0 > 3) sel0 = (~sel0 >> 31) & 3; + if ((uint32_t)sel1 > 3) sel1 = (~sel1 >> 31) & 3; + if ((uint32_t)sel2 > 3) sel2 = (~sel2 >> 31) & 3; + if ((uint32_t)sel3 > 3) sel3 = (~sel3 >> 31) & 3; + + pWeights1[i + 0] = (uint8_t)sel0; + pWeights1[i + 1] = (uint8_t)sel1; + pWeights1[i + 2] = (uint8_t)sel2; + pWeights1[i + 3] = (uint8_t)sel3; + + sse += bc7_sse(pPixels[i + 0][3], la, da, basist::g_bc7_weights2[sel0]); + sse += bc7_sse(pPixels[i + 1][3], la, da, basist::g_bc7_weights2[sel1]); + sse += bc7_sse(pPixels[i + 2][3], la, da, basist::g_bc7_weights2[sel2]); + sse += bc7_sse(pPixels[i + 3][3], la, da, basist::g_bc7_weights2[sel3]); + } + + return sse; + } + + // Determines the best unique pbits to use to encode xl/xh, which are [0,1] + static void determine_unique_pbits( + uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], + color_rgba& bestMinColor, color_rgba& bestMaxColor, uint32_t best_pbits[2]) + { +#ifdef _DEBUG + for (uint32_t c = 0; c < total_comps; c++) + { + assert((xl[c] >= 0.0f) && (xl[c] <= 1.0f)); + assert((xh[c] >= 0.0f) && (xh[c] <= 1.0f)); + } +#endif + + const uint32_t total_bits = comp_bits + 1; + const int iscalep = (1 << total_bits) - 1; + const float scalep = (float)iscalep; + + float best_err0 = 1e+9f; + float best_err1 = 1e+9f; + + for (int p = 0; p < 2; p++) + { + color_rgba xMinColor, xMaxColor; + + for (uint32_t c = 0; c < 4; c++) + { + xMinColor[c] = (uint8_t)(clampi(((int)((xl[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + xMaxColor[c] = (uint8_t)(clampi(((int)((xh[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + } + + color_rgba scaledLow, scaledHigh; + for (uint32_t i = 0; i < 4; i++) + { + scaledLow[i] = (xMinColor[i] << (8 - total_bits)); + scaledLow[i] |= (scaledLow[i] >> total_bits); + assert(scaledLow[i] <= 255); + + scaledHigh[i] = (xMaxColor[i] << (8 - total_bits)); + scaledHigh[i] |= (scaledHigh[i] >> total_bits); + assert(scaledHigh[i] <= 255); + } + + float err0 = 0, err1 = 0; + for (uint32_t i = 0; i < total_comps; i++) + { + err0 += basisu::squaref(scaledLow[i] - xl[i] * 255.0f); + err1 += basisu::squaref(scaledHigh[i] - xh[i] * 255.0f); + } + + if (err0 < best_err0) + { + best_err0 = err0; + best_pbits[0] = p; + + bestMinColor[0] = xMinColor[0] >> 1; + bestMinColor[1] = xMinColor[1] >> 1; + bestMinColor[2] = xMinColor[2] >> 1; + bestMinColor[3] = xMinColor[3] >> 1; + } + + if (err1 < best_err1) + { + best_err1 = err1; + best_pbits[1] = p; + + bestMaxColor[0] = xMaxColor[0] >> 1; + bestMaxColor[1] = xMaxColor[1] >> 1; + bestMaxColor[2] = xMaxColor[2] >> 1; + bestMaxColor[3] = xMaxColor[3] >> 1; + } + } + } + + // Determines the best shared pbits to use to encode xl/xh, which are [0,1] + static void determine_shared_pbits( + uint32_t total_comps, uint32_t comp_bits, float xl[4], float xh[4], + color_rgba& bestMinColor, color_rgba& bestMaxColor, uint32_t best_pbits[2]) + { +#ifdef _DEBUG + for (uint32_t c = 0; c < total_comps; c++) + { + assert((xl[c] >= 0.0f) && (xl[c] <= 1.0f)); + assert((xh[c] >= 0.0f) && (xh[c] <= 1.0f)); + } +#endif + + const uint32_t total_bits = comp_bits + 1; + assert((total_bits >= 4) && (total_bits <= 8)); + + const int iscalep = (1 << total_bits) - 1; + const float scalep = (float)iscalep; + + float best_err = 1e+9f; + + for (int p = 0; p < 2; p++) + { + color_rgba xMinColor, xMaxColor; + for (uint32_t c = 0; c < 4; c++) + { + xMinColor[c] = (uint8_t)(clampi(((int)((xl[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + xMaxColor[c] = (uint8_t)(clampi(((int)((xh[c] * scalep - p) * (1.0f / 2.0f) + .5f)) * 2 + p, p, iscalep - 1 + p)); + } + + color_rgba scaledLow, scaledHigh; + + for (uint32_t i = 0; i < 4; i++) + { + scaledLow[i] = (xMinColor[i] << (8 - total_bits)); + scaledLow[i] |= (scaledLow[i] >> total_bits); + assert(scaledLow[i] <= 255); + + scaledHigh[i] = (xMaxColor[i] << (8 - total_bits)); + scaledHigh[i] |= (scaledHigh[i] >> total_bits); + assert(scaledHigh[i] <= 255); + } + + float err = 0; + for (uint32_t i = 0; i < total_comps; i++) + err += basisu::squaref((scaledLow[i] * (1.0f / 255.0f)) - xl[i]) + basisu::squaref((scaledHigh[i] * (1.0f / 255.0f)) - xh[i]); + + if (err < best_err) + { + best_err = err; + best_pbits[0] = p; + best_pbits[1] = p; + for (uint32_t j = 0; j < 4; j++) + { + bestMinColor[j] = xMinColor[j] >> 1; + bestMaxColor[j] = xMaxColor[j] >> 1; + } + } + } + } + + // 4x4 ASTC blocks only, no dp, no subsets, outputs mode 6 + static void pack_from_astc_4x4_single_subset(uint8_t* pDst_block_u8, const astc_helpers::log_astc_block& log_blk) + { + assert(!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1)); + assert((log_blk.m_grid_width <= 4) && (log_blk.m_grid_height <= 4)); + + color_rgba l, h; + astc_ldr_t::decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + uint8_t dequantized_weights[16]; + uint8_t upsampled_weights[16]; + + const uint32_t total_weight_vals = log_blk.m_grid_width * log_blk.m_grid_height; + + const astc_helpers::dequant_table& weight_dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + + dequantized_weights[i] = pWeight_dequant[log_blk.m_weights[i]]; + } + + const uint8_t* pUpsampled_weights = dequantized_weights; + if ((log_blk.m_grid_width < 4) || (log_blk.m_grid_height < 4)) + { + astc_helpers::upsample_weight_grid_xuastc_ldr(4, 4, log_blk.m_grid_width, log_blk.m_grid_height, dequantized_weights, upsampled_weights, nullptr, nullptr); + pUpsampled_weights = upsampled_weights; + } + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + uint8_t bc7_weights[16]; + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t i = 0; i < 16; i++) + bc7_weights[i] = (uint8_t)((pUpsampled_weights[i] * 15 + 32) >> 6); + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + // Outputs mode 6 + static void pack_from_astc_single_subset( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& log_blk, + const uint8_t *pUpsampled_weights, + uint32_t weight_ofs_x, uint32_t weight_ofs_y, + uint32_t block_width, uint32_t block_height) + { + BASISU_NOTE_UNUSED(block_width); + BASISU_NOTE_UNUSED(block_height); + + assert(!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1)); + assert((log_blk.m_grid_width <= block_width) && (log_blk.m_grid_height <= block_height)); + assert((weight_ofs_x + 3) < block_width); + assert((weight_ofs_y + 3) < block_height); + + color_rgba l, h; + astc_ldr_t::decode_endpoints(log_blk.m_color_endpoint_modes[0], log_blk.m_endpoints, log_blk.m_endpoint_ise_range, l, h); + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w = pUpsampled_weights[(weight_ofs_x + x) + (weight_ofs_y + y) * block_width]; + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 15 + 32) >> 6); + } // x + } // y + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + // same or super close endpoints, 8x6 or 6x6 only + void pack_from_astc_to_single_subset_same_endpoints( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy, + uint32_t block_width, uint32_t block_height) + { + assert(!b0.m_dual_plane && (b0.m_num_partitions == 1)); + assert((b0.m_grid_width <= block_width) && (b0.m_grid_height <= block_height)); + assert(!b0.m_solid_color_flag_ldr); + + assert(!b1.m_dual_plane && (b1.m_num_partitions == 1)); + assert((b1.m_grid_width <= block_width) && (b1.m_grid_height <= block_height)); + assert(!b1.m_solid_color_flag_ldr); + + const bool is_6x6 = ((block_width == 6) && (block_height == 6)); + + // Only handles particular BC7 blocks in the 3x3 or 2x3 region. + if (is_6x6) + { + // 6x6 + assert( + (!dx && (dy == 1)) || ((dx == 2) && (dy == 1)) || + ((dx == 1) && !dy) || ((dx == 1) && (dy == 2)) + ); + } + else + { + // 8x6 + assert((block_width == 8) && (block_height == 6)); + assert((dx >= 0) && (dx <= 1)); + assert((dy >= 0) && (dy <= 2)); + } + + color_rgba l, h; + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l, h); + + color_rgba al, ah; + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, al, ah); + + for (uint32_t c = 0; c < 4; c++) + { + l[c] = (l[c] + al[c] + 1) >> 1; + h[c] = (h[c] + ah[c] + 1) >> 1; + } + + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)l.r * q, (float)l.g * q, (float)l.b * q, (float)l.a * q }; + float sxh[4] = { (float)h.r * q, (float)h.g * q, (float)h.b * q, (float)h.a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + bool top_or_bottom = false; + + if (is_6x6) + top_or_bottom = (dy == 0) || (dy == 2); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (is_6x6) + { + if (top_or_bottom) + { + if (x < 2) + w = pUpsampled_weights0[basisu::open_range_check((x + 4) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + else + w = pUpsampled_weights1[basisu::open_range_check((x - 2) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + } + else + { + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y + 4) * 6, 0, 36)]; + else + w = pUpsampled_weights1[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y - 2) * 6, 0, 36)]; + } + } + else + { + // 8x6 + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + } + + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 15 + 32) >> 6); + } // x + } // y + + encode_mode6_rgba_block(pDst_block_u8, + bestMinColor.r, bestMinColor.g, bestMinColor.b, bestMinColor.a, best_pbits[0], + bestMaxColor.r, bestMaxColor.g, bestMaxColor.b, bestMaxColor.a, best_pbits[1], + bc7_weights); + } + + bool pack_from_astc_6x6_to_two_subsets_different_endpoints( + uint8_t *pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy) + { + const bool b0_solid = b0.m_solid_color_flag_ldr; + const bool b1_solid = b1.m_solid_color_flag_ldr; + + assert(b0_solid || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 6) && (b0.m_grid_height <= 6)); + + assert(b1_solid || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 6) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 3x3 region. + assert( + (!dx && (dy == 1)) || ((dx == 2) && (dy == 1)) || + ((dx == 1) && !dy) || ((dx == 1) && (dy == 2)) + ); + + color_rgba l[2], h[2]; + if (b0_solid) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b1_solid) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = l[i][j] * q; + sxh[i][j] = h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + const bool top_or_bottom = (dy == 0) || (dy == 2); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 0; + if ((dx == 0) || (dx == 2)) + part_id = 13; + +#if 0 + const uint8_t* pPart_map = &g_bc7_partition2[part_id * 16]; + uint32_t min_qw[2] = { 256, 256 }, max_qw[2] = { 0, 0 }; +#endif + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (top_or_bottom) + { + if (x < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((x + 4) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((x - 2) + (y + ((dy == 2) ? 2 : 0)) * 6, 0, 36)]; + } + else + { + if (y < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y + 4) * 6, 0, 36)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((x + ((dx == 2) ? 2 : 0)) + (y - 2) * 6, 0, 36)]; + } + + assert(w <= 64); + + uint32_t qw = ((w * 7 + 32) >> 6); + +#if 0 + const uint8_t s = pPart_map[x + y * 4]; + min_qw[s] = basisu::minimum(min_qw[s], qw); + max_qw[s] = basisu::maximum(max_qw[s], qw); +#endif + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + +#if 0 + const uint32_t w_range_0 = max_qw[0] - min_qw[0]; + const uint32_t w_range_1 = max_qw[1] - min_qw[1]; + const uint32_t W_RANGE_THRESH = 2; + if ((w_range_0 <= W_RANGE_THRESH) || (w_range_1 <= W_RANGE_THRESH)) + return false; +#endif + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + bool pack_from_astc_8x6_to_two_subsets_different_endpoints( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy) + { + BASISU_NOTE_UNUSED(dy); + + const bool b0_solid = b0.m_solid_color_flag_ldr; + const bool b1_solid = b1.m_solid_color_flag_ldr; + + assert(b0_solid || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 8) && (b0.m_grid_height <= 6)); + + assert(b1_solid || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 8) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 2x3 region. + assert((dx >= 0) && (dx <= 1) && + (dy >= 0) && (dy <= 2)); + + color_rgba l[2], h[2]; + if (b0_solid) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b1_solid) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = l[i][j] * q; + sxh[i][j] = h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 13; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w; + + if (y < 2) + w = b0_solid ? 0 : pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = b1_solid ? 0 : pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + assert(w <= 64); + + uint32_t qw = ((w * 7 + 32) >> 6); + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + bool pack_from_astc_8x6_to_two_subsets_different_endpoints_hq( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block& b0, const uint8_t* pUpsampled_weights0, + const astc_helpers::log_astc_block& b1, const uint8_t* pUpsampled_weights1, + int dx, int dy, bool astc_srgb_decode, bool &fallback_encode_flag) + { + BASISU_NOTE_UNUSED(dy); + + const bool b_solid[2] = { b0.m_solid_color_flag_ldr, b1.m_solid_color_flag_ldr }; + + assert(b_solid[0] || (!b0.m_dual_plane && (b0.m_num_partitions == 1))); + assert((b0.m_grid_width <= 8) && (b0.m_grid_height <= 6)); + + assert(b_solid[1] || (!b1.m_dual_plane && (b1.m_num_partitions == 1))); + assert((b1.m_grid_width <= 8) && (b1.m_grid_height <= 6)); + + // Only handles particular BC7 blocks in the 2x3 region. + assert((dx >= 0) && (dx <= 1) && + (dy >= 0) && (dy <= 2)); + + color_rgba l[2], h[2]; + if (b_solid[0]) + { + l[0][0] = h[0][0] = (uint8_t)(b0.m_solid_color[0] >> 8); + l[0][1] = h[0][1] = (uint8_t)(b0.m_solid_color[1] >> 8); + l[0][2] = h[0][2] = (uint8_t)(b0.m_solid_color[2] >> 8); + l[0][3] = h[0][3] = (uint8_t)(b0.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b0.m_color_endpoint_modes[0], b0.m_endpoints, b0.m_endpoint_ise_range, l[0], h[0]); + } + + if (b_solid[1]) + { + l[1][0] = h[1][0] = (uint8_t)(b1.m_solid_color[0] >> 8); + l[1][1] = h[1][1] = (uint8_t)(b1.m_solid_color[1] >> 8); + l[1][2] = h[1][2] = (uint8_t)(b1.m_solid_color[2] >> 8); + l[1][3] = h[1][3] = (uint8_t)(b1.m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(b1.m_color_endpoint_modes[0], b1.m_endpoints, b1.m_endpoint_ise_range, l[1], h[1]); + } + + uint32_t low_w[2] = { UINT32_MAX, UINT32_MAX }; + uint32_t high_w[2] = { 0, 0 }; + + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s; + uint32_t w; + + if (y < 2) + { + w = b_solid[0] ? 0 : pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + s = 0; + } + else + { + w = b_solid[1] ? 0 : pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + s = 1; + } + + assert(w <= 64); + + low_w[s] = basisu::minimum(low_w[s], w); + high_w[s] = basisu::maximum(high_w[s], w); + } // x + } // y + + color_rgba orig_l[2], orig_h[2]; + memcpy(orig_l, l, sizeof(l)); + memcpy(orig_h, h, sizeof(h)); + +#if 1 + uint32_t num_low_stddev = 0; +#endif + + for (uint32_t s = 0; s < 2; s++) + { + if (b_solid[s]) + continue; + + if ((low_w[s] > 0) || (high_w[s] < 64)) + { + for (uint32_t c = 0; c < 3; c++) + { + l[s][c] = (uint8_t)astc_helpers::channel_interpolate(orig_l[s][c], orig_h[s][c], low_w[s], astc_srgb_decode); + h[s][c] = (uint8_t)astc_helpers::channel_interpolate(orig_l[s][c], orig_h[s][c], high_w[s], astc_srgb_decode); + } + } + +#if 1 + uint32_t e_delta = basisu::squarei((int)h[s][0] - (int)l[s][0]) + + basisu::squarei((int)h[s][1] - (int)l[s][1]) + + basisu::squarei((int)h[s][2] - (int)l[s][2]); + + const uint32_t E_DELTA_THRESH = 60; + num_low_stddev += (e_delta < E_DELTA_THRESH); +#endif + } + +#if 1 + if (num_low_stddev == 2) + { + //bc7f::pack_mode5_solid(pDst_block_u8, color_rgba(200, 0, 0, 255)); + //return true; + + assert(!b_solid[0] && !b_solid[1]); + + color_rgba dec_pixels[16]; + + int ep_l[2][3], ep_h[2][3]; + for (uint32_t s = 0; s < 2; s++) + { + for (uint32_t c = 0; c < 3; c++) + { + int le = l[s][c], he = h[s][c]; + + if (astc_srgb_decode) + { + le = (le << 8) | 0x80; + he = (he << 8) | 0x80; + } + else + { + le = (le << 8) | le; + he = (he << 8) | he; + } + + ep_l[s][c] = le; + ep_h[s][c] = he; + } + } + + color_rgba* pDst = dec_pixels; + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s = (y < 2) ? 0 : 1; + + int w; + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + pDst->r = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][0], ep_h[s][0], w) >> 8); + pDst->g = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][1], ep_h[s][1], w) >> 8); + pDst->b = (uint8_t)(astc_helpers::weight_interpolate(ep_l[s][2], ep_h[s][2], w) >> 8); + pDst->a = 255; + + ++pDst; + + } // x + } // y + + const uint32_t flags = cPackBC7FlagUseTrivialMode6 | cPackBC7FlagPBitOptMode6; + //const uint32_t flags = cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6 | cPackBC7FlagUseTrivialMode6; + bc7f::fast_pack_bc7_rgb_analytical(pDst_block_u8, dec_pixels, flags); + fallback_encode_flag = true; + return true; + } +#endif + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float q = 1.0f / 255.0f; + + sxl[i][j] = (float)l[i][j] * q; + sxh[i][j] = (float)h[i][j] * q; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + uint32_t part_id = 13; + + float one_over_w_range_scaled[2]; + for (uint32_t s = 0; s < 2; s++) + { + if (low_w[s] == high_w[s]) + one_over_w_range_scaled[s] = 0; + else + one_over_w_range_scaled[s] = 7.0f / (high_w[s] - low_w[s]); + } + + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t s = (y < 2) ? 0 : 1; + + int qw = 0; + + if (!b_solid[s]) + { + int w; + + if (y < 2) + w = pUpsampled_weights0[basisu::open_range_check((dx * 4 + x) + (y + 4) * 8, 0, 48)]; + else + w = pUpsampled_weights1[basisu::open_range_check((dx * 4 + x) + (y - 2) * 8, 0, 48)]; + + assert(w <= 64); + + if (low_w[s] != high_w[s]) + { + float f = ((float)w - (float)low_w[s]) * one_over_w_range_scaled[s]; + + qw = (int)(f + .5f); + + if ((uint32_t)qw > 7) + { + qw = basisu::clamp(qw, 0, 7); + } + } + } + + bc7_weights[x + y * 4] = (uint8_t)qw; + } // x + } // y + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + + return true; + } + + void pack_astc_6x6_to_two_subsets_middle_block( + uint8_t* pDst_block_u8, + const astc_helpers::log_astc_block* blocks[2][2], + const uint8_t(&weights)[2][2][36], + bool do_left_right) + { + const astc_helpers::log_astc_block* p[2]; + const astc_helpers::log_astc_block* q[2]; + + if (do_left_right) + { + // left and right into separate subsets + p[0] = blocks[0][0]; + q[0] = blocks[0][1]; + + p[1] = blocks[1][0]; + q[1] = blocks[1][1]; + } + else + { + // top and bottom into separate subsets + p[0] = blocks[0][0]; + q[0] = blocks[1][0]; + + p[1] = blocks[0][1]; + q[1] = blocks[1][1]; + } + + assert(p[0]->m_solid_color_flag_ldr || (!p[0]->m_dual_plane && (p[0]->m_num_partitions == 1))); + assert((p[0]->m_grid_width <= 6) && (p[0]->m_grid_height <= 6)); + + assert(p[1]->m_solid_color_flag_ldr || (!p[1]->m_dual_plane && (p[1]->m_num_partitions == 1))); + assert((p[1]->m_grid_width <= 6) && (p[1]->m_grid_height <= 6)); + + assert(q[0]->m_solid_color_flag_ldr || (!q[0]->m_dual_plane && (q[0]->m_num_partitions == 1))); + assert((q[0]->m_grid_width <= 6) && (q[0]->m_grid_height <= 6)); + + assert(q[1]->m_solid_color_flag_ldr || (!q[1]->m_dual_plane && (q[1]->m_num_partitions == 1))); + assert((q[1]->m_grid_width <= 6) && (q[1]->m_grid_height <= 6)); + + color_rgba el[2], eh[2]; + color_rgba el2[2], eh2[2]; + + for (uint32_t i = 0; i < 2; i++) + { + if (p[i]->m_solid_color_flag_ldr) + { + el[i][0] = eh[i][0] = (uint8_t)(p[i]->m_solid_color[0] >> 8); + el[i][1] = eh[i][1] = (uint8_t)(p[i]->m_solid_color[1] >> 8); + el[i][2] = eh[i][2] = (uint8_t)(p[i]->m_solid_color[2] >> 8); + el[i][3] = eh[i][3] = (uint8_t)(p[i]->m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(p[i]->m_color_endpoint_modes[0], p[i]->m_endpoints, p[i]->m_endpoint_ise_range, el[i], eh[i]); + } + + if (q[i]->m_solid_color_flag_ldr) + { + el2[i][0] = eh2[i][0] = (uint8_t)(q[i]->m_solid_color[0] >> 8); + el2[i][1] = eh2[i][1] = (uint8_t)(q[i]->m_solid_color[1] >> 8); + el2[i][2] = eh2[i][2] = (uint8_t)(q[i]->m_solid_color[2] >> 8); + el2[i][3] = eh2[i][3] = (uint8_t)(q[i]->m_solid_color[3] >> 8); + } + else + { + astc_ldr_t::decode_endpoints(q[i]->m_color_endpoint_modes[0], q[i]->m_endpoints, q[i]->m_endpoint_ise_range, el2[i], eh2[i]); + } + + assert(el[i][3] == 255); + assert(el2[i][3] == 255); + + assert(eh[i][3] == 255); + assert(eh2[i][3] == 255); + } + + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t c = 0; c < 3; c++) + { + el[i][c] = (el[i][c] + el2[i][c] + 1) >> 1; + eh[i][c] = (eh[i][c] + eh2[i][c] + 1) >> 1; + } // c + } // i + + float sxl[2][4], sxh[2][4]; + for (uint32_t i = 0; i < 2; i++) + { + for (uint32_t j = 0; j < 4; j++) + { + const float S = 1.0f / 255.0f; + + sxl[i][j] = el[i][j] * S; + sxh[i][j] = eh[i][j] * S; + } // j + } // i + + color_rgba bestMinColor[2], bestMaxColor[2]; + + uint32_t best_p0[2]; + determine_shared_pbits(3, 6, sxl[0], &sxh[0][0], bestMinColor[0], bestMaxColor[0], best_p0); + + uint32_t best_p1[2]; + determine_shared_pbits(3, 6, sxl[1], &sxh[1][0], bestMinColor[1], bestMaxColor[1], best_p1); + + uint8_t bc7_weights[16]; + + // TODO: Potentially improve this mapping using a lookup table + for (uint32_t y = 0; y < 4; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + uint32_t w = 0; + + if (y < 2) + { + if (x < 2) + { + if (!blocks[0][0]->m_solid_color_flag_ldr) + w = weights[0][0][(x + 4) + (y + 4) * 6]; + } + else + { + if (!blocks[1][0]->m_solid_color_flag_ldr) + w = weights[1][0][(x - 2) + (y + 4) * 6]; + } + } + else + { + if (x < 2) + { + if (!blocks[0][1]->m_solid_color_flag_ldr) + w = weights[0][1][(x + 4) + (y - 2) * 6]; + } + else + { + if (!blocks[1][1]->m_solid_color_flag_ldr) + w = weights[1][1][(x - 2) + (y - 2) * 6]; + } + } + + assert(w <= 64); + + bc7_weights[x + y * 4] = (uint8_t)((w * 7 + 32) >> 6); + } // x + } // y + + uint32_t part_id = 13; + if (do_left_right) + part_id = 0; + + uint32_t lr[2] = { bestMinColor[0][0], bestMinColor[1][0] }; + uint32_t lg[2] = { bestMinColor[0][1], bestMinColor[1][1] }; + uint32_t lb[2] = { bestMinColor[0][2], bestMinColor[1][2] }; + + uint32_t hr[2] = { bestMaxColor[0][0], bestMaxColor[1][0] }; + uint32_t hg[2] = { bestMaxColor[0][1], bestMaxColor[1][1] }; + uint32_t hb[2] = { bestMaxColor[0][2], bestMaxColor[1][2] }; + + encode_mode1_rgb_block(pDst_block_u8, part_id, + lr, lg, lb, + hr, hg, hb, + best_p0[0], best_p1[0], + bc7_weights); + } + +#if 0 + // var must be variance (divided by N, # pixels), not SSE + static inline int calc_span_est(int min_c, int max_c, int mean_c, float var) + { + // variance-implied span: span_var = ~sqrt(12 * var) + int span_var = (int)fast_roundf_pos_int(std::sqrtf((float)(12.0f * var))); + + // take into account available headroom on the low/high end + span_var = basisu::minimum(span_var, 2 * basisu::minimum(mean_c, 255 - mean_c)); + + return basisu::minimum(max_c - min_c, span_var); + } +#endif + + // Multi-channel estimates + // returns total SSE (pixel SSE * num_pixels), span_weights can be nullptr + float analytical_quant_est_sse(int e_levels, int w_levels, int num_chans, const int spans[4], const float span_weights[4], float endpoint_weight_scale, int num_pixels) + { + assert((e_levels >= 2) && (e_levels <= 256) && (w_levels >= 2) && (num_chans)); + assert(spans); + + const float Dep = 1.0f / (float)(e_levels - 1); // endpoint quant step + const float Dw = 1.0f / (float)(w_levels - 1); // weight quant step + + // TODO: precompute + const float N = float(w_levels); + const float ab_sum = (2.0f * N - 1.0f) / (3.0f * (N - 1.0f)); + + float pixel_sse = (e_levels == 256) ? 0.0f : ((Dep * Dep) * ((1.0f / 12.0f) * ab_sum * (255.0f * 255.0f)) * (float)num_chans * endpoint_weight_scale); + + const float k = (Dw * Dw) * (1.0f / 12.0f); + for (int i = 0; i < num_chans; i++) + { + pixel_sse += k * (float)(spans[i] * spans[i]) * (span_weights ? span_weights[i] : 1.0f); + } + + return pixel_sse * float(num_pixels); + } + + // Single channel estimates + float analytical_quant_est_sse(int e_levels, int w_levels, int span, float span_weight, float endpoint_weight_scale, int num_pixels) + { + assert((e_levels >= 2) && (e_levels <= 256) && (w_levels >= 2)); + + const float Dep = 1.0f / (float)(e_levels - 1); // endpoint quant step + const float Dw = 1.0f / (float)(w_levels - 1); // weight quant step + + // TODO: precompute + const float N = float(w_levels); + const float ab_sum = (2.0f * N - 1.0f) / (3.0f * (N - 1.0f)); + + float pixel_sse = (e_levels == 256) ? 0.0f : ((Dep * Dep) * ((1.0f / 12.0f) * ab_sum * (255.0f * 255.0f)) * endpoint_weight_scale); + + pixel_sse += (Dw * Dw) * (1.0f / 12.0f) * (float)(span * span) * span_weight; + + return pixel_sse * float(num_pixels); + } + + // if cov[] wasn't divided by the # of pixels, this is SSE + float estimate_slam_to_line_sse_3D(const float cov[6], float xr, float yr, float zr, float* pOrtho_ratio = nullptr) + { + // total var + const float total_var = cov[0] + cov[3] + cov[5]; + + float l = sqrtf(xr * xr + yr * yr + zr * zr); + if (l < basisu::SMALL_FLOAT_VAL) + { + xr = yr = zr = 0.577350269f; + } + else + { + l = 1.0f / l; + xr *= l; yr *= l; zr *= l; + } + + float xr2 = cov[0] * xr + cov[1] * yr + cov[2] * zr; + float xg2 = cov[1] * xr + cov[3] * yr + cov[4] * zr; + float xb2 = cov[2] * xr + cov[4] * yr + cov[5] * zr; + + // Rayleigh quotient/est var of principal axis + const float principal_axis_var = xr2 * xr + xg2 * yr + xb2 * zr; + + // Compute leftover var, this is the var unexplaind by the principal axis + const float ortho_var = basisu::maximum(0.0f, total_var - principal_axis_var); + + if (pOrtho_ratio) + *pOrtho_ratio = (total_var > basisu::SMALL_FLOAT_VAL) ? (ortho_var / total_var) : 0.0f; + + return ortho_var; + } + + float estimate_slam_to_line_sse_4D(const float cov[10], float xr, float yr, float zr, float wr, float* pOrtho_ratio = nullptr) + { + // total var + const float total_var = cov[0] + cov[4] + cov[7] + cov[9]; + + float l = sqrtf(xr * xr + yr * yr + zr * zr + wr * wr); + if (l < basisu::SMALL_FLOAT_VAL) + { + xr = yr = zr = wr = .5f; + } + else + { + l = 1.0f / l; + xr *= l; yr *= l; zr *= l; wr *= l; + } + + float xr2 = cov[0] * xr + cov[1] * yr + cov[2] * zr + cov[3] * wr; + float xg2 = cov[1] * xr + cov[4] * yr + cov[5] * zr + cov[6] * wr; + float xb2 = cov[2] * xr + cov[5] * yr + cov[7] * zr + cov[8] * wr; + float xa2 = cov[3] * xr + cov[6] * yr + cov[8] * zr + cov[9] * wr; + + // Rayleigh quotient/est var of principal axis + const float principal_axis_var = xr2 * xr + xg2 * yr + xb2 * zr + xa2 * wr; + + // Compute leftover var, this is the var unexplaind by the principal axis + const float ortho_var = basisu::maximum(0.0f, total_var - principal_axis_var); + + if (pOrtho_ratio) + *pOrtho_ratio = (total_var > basisu::SMALL_FLOAT_VAL) ? (ortho_var / total_var) : 0.0f; + + return ortho_var; + } + + uint32_t calc_sse(const uint8_t* pBlock, const color_rgba* pPixels) + { + color_rgba unpacked_pixels[16]; + bool status = bc7u::unpack_bc7(pBlock, unpacked_pixels); + if (!status) + { + assert(0); + return UINT32_MAX; + } + + uint32_t sse = 0; + for (uint32_t i = 0; i < 16; i++) + sse += basisu::squarei(pPixels[i][0] - unpacked_pixels[i][0]) + basisu::squarei(pPixels[i][1] - unpacked_pixels[i][1]) + basisu::squarei(pPixels[i][2] - unpacked_pixels[i][2]) + basisu::squarei(pPixels[i][3] - unpacked_pixels[i][3]); + + return sse; + } + + bool pack_mode1_or_3_rgb(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b, + float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_evals++; +#endif + + uint32_t desired_pat_bits = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const float r = (float)(pPixels[i].r - block_mean_r); + const float g = (float)(pPixels[i].g - block_mean_g); + const float b = (float)(pPixels[i].b - block_mean_b); + + const uint32_t subset = (r * block_xr + g * block_xg + b * block_xb) > 0.0f; + + desired_pat_bits |= (subset << i); + } + + uint32_t best_diff = UINT32_MAX; + for (uint32_t p = 0; p < MAX_PATTERNS2_TO_CHECK; p++) + { + const uint32_t bc6h_pat_bits = g_bc7_part2_bitmasks[p]; + + int diff = popcount32(bc6h_pat_bits ^ desired_pat_bits); + int diff_inv = 16 - diff; + + uint32_t min_diff = (basisu::minimum(diff, diff_inv) << 8) | p; + if (min_diff < best_diff) + best_diff = min_diff; + } // p + + const uint32_t best_pat_index = best_diff & 0xFF; + const uint32_t best_pat_bits = g_bc7_part2_bitmasks[best_pat_index]; + + int total_r[2] = { }, total_g[2] = { }, total_b[2] = { }, total_c[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int subset = (best_pat_bits >> i) & 1; + + total_r[subset] += r; total_g[subset] += g; total_b[subset] += b; + total_c[subset]++; + } + + int mean_r[2], mean_g[2], mean_b[2]; + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t t = total_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (total_r[s] + h) / t; + mean_g[s] = (total_g[s] + h) / t; + mean_b[s] = (total_b[s] + h) / t; + } + + int icov[2][6] = { { }, { } }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + + int r = (int)pPixels[i].r - mean_r[subset]; + int g = (int)pPixels[i].g - mean_g[subset]; + int b = (int)pPixels[i].b - mean_b[subset]; + icov[subset][0] += r * r; icov[subset][1] += r * g; icov[subset][2] += r * b; + icov[subset][3] += g * g; icov[subset][4] += g * b; + icov[subset][5] += b * b; + } + + int ar[2], ag[2], ab[2]; + + // Slam to line SSE estimate is the same for both mode 1 and 3. + float slam_to_line_sse_est = 0.0f; + + for (uint32_t s = 0; s < 2; s++) + { + int block_max_var = basisu::maximum(icov[s][0], icov[s][3], icov[s][5]); + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[s][i]; + + const float sc = 1.0f / ((float)block_max_var + .0000125f); + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + slam_to_line_sse_est += estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + } // s + + int low_dot[2] = { INT_MAX, INT_MAX }; + int high_dot[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset]; + + int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + int low_c[2] = { low_dot[0] & 15, low_dot[1] & 15 }; + int high_c[2] = { high_dot[0] & 15, high_dot[1] & 15 }; + + int spans[4]; + spans[3] = 0; + + // Endpoint/weight quant error estimates for modes 1 and 3 + float quant_err_sse_est[2] = { }; + + for (uint32_t subset = 0; subset < 2; subset++) + { + const uint32_t low_pixel = low_c[subset]; + const uint32_t high_pixel = high_c[subset]; + + for (uint32_t c = 0; c < 3; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + // mode 1: 6-bit endpoints, unique pbits, 3 bit weights + quant_err_sse_est[0] += analytical_quant_est_sse(64, 8, 3, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? UNIQUE_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + // mode 3, 7-bit endpoints, shared pbits, 2-bit weights + quant_err_sse_est[1] += analytical_quant_est_sse(128, 4, 3, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? SHARED_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + } // subset + + const float total_mode1_est_sse = slam_to_line_sse_est + quant_err_sse_est[0]; + const float total_mode3_est_sse = slam_to_line_sse_est + quant_err_sse_est[1]; + + if (total_mode1_est_sse < total_mode3_est_sse) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode1_est_sse; + + // Mode 1: Large span + if (total_mode1_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2]; + uint32_t hr[2], hg[2], hb[2]; + uint32_t pbits[2] = { 0, 0 }; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s], hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, 0 }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_shared_pbits(3, 6, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s] = best_pbits[0]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + int l = pPixels[lc].r + pPixels[lc].g + pPixels[lc].b; + int h = pPixels[hc].r + pPixels[hc].g + pPixels[hc].b; + + if (basisu::maximum(l, h) >= 129 * 3) + pbits[s] = 1; + + lr[s] = to_6(pPixels[lc].r, pbits[s]); + lg[s] = to_6(pPixels[lc].g, pbits[s]); + lb[s] = to_6(pPixels[lc].b, pbits[s]); + + hr[s] = to_6(pPixels[hc].r, pbits[s]); + hg[s] = to_6(pPixels[hc].g, pbits[s]); + hb[s] = to_6(pPixels[hc].b, pbits[s]); + } + } // s + + uint8_t cur_weights[16]; + eval_weights_mode1_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 7); + + z00[subset] += g_bc7_3bit_ls_tab[sel][0]; + z10[subset] += g_bc7_3bit_ls_tab[sel][1]; + z11[subset] += g_bc7_3bit_ls_tab[sel][2]; + + const float w = g_bc7_3bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), 0 }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_shared_pbits(3, 6, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s] = best_pbits[0]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const float l = slr + slg + slb, h = shr + shg + shb; + + pbits[s] = (basisu::maximum(l, h) >= 129.0f * 3.0f); + + lr[s] = to_6_clamp(slr, pbits[s]); + hr[s] = to_6_clamp(shr, pbits[s]); + + lg[s] = to_6_clamp(slg, pbits[s]); + hg[s] = to_6_clamp(shg, pbits[s]); + + lb[s] = to_6_clamp(slb, pbits[s]); + hb[s] = to_6_clamp(shb, pbits[s]); + } + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode1_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + else + eval_weights_mode1_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + encode_mode1_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, pbits[0], pbits[1], cur_weights); + } + else + { + // Mode 3: Small span + if (pFinal_sse_est) + *pFinal_sse_est = total_mode3_est_sse; + + if (total_mode3_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode13_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2]; + uint32_t hr[2], hg[2], hb[2]; + uint32_t pbits[4]; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s]; + const int hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, 0 }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s * 2 + 0] = best_pbits[0]; + pbits[s * 2 + 1] = best_pbits[1]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const int l = pPixels[lc].r + pPixels[lc].g + pPixels[lc].b; + const int l_pbit = (l >= 129); + pbits[s * 2 + 0] = l_pbit; + + lr[s] = to_7(pPixels[lc].r, l_pbit); + lg[s] = to_7(pPixels[lc].g, l_pbit); + lb[s] = to_7(pPixels[lc].b, l_pbit); + + int h = pPixels[hc].r + pPixels[hc].g + pPixels[hc].b; + const int h_pbit = (h >= 129); + pbits[s * 2 + 1] = h_pbit; + + hr[s] = to_7(pPixels[hc].r, h_pbit); + hg[s] = to_7(pPixels[hc].g, h_pbit); + hb[s] = to_7(pPixels[hc].b, h_pbit); + } + } // s + + uint8_t cur_weights[16]; + eval_weights_mode3_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), 0 }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + pbits[s * 2 + 0] = best_pbits[0]; + pbits[s * 2 + 1] = best_pbits[1]; + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; + } + else + { + const float l = slr + slg + slb; + const int l_pbit = (l >= 129.0f * 3.0f); + pbits[s * 2 + 0] = l_pbit; + + lr[s] = to_7_clamp(slr, l_pbit); + lg[s] = to_7_clamp(slg, l_pbit); + lb[s] = to_7_clamp(slb, l_pbit); + + const float h = shr + shg + shb; + const int h_pbit = (h >= 129.0f * 3.0f); + pbits[s * 2 + 1] = h_pbit; + + hr[s] = to_7_clamp(shr, h_pbit); + hg[s] = to_7_clamp(shg, h_pbit); + hb[s] = to_7_clamp(shb, h_pbit); + } + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode3_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + else + eval_weights_mode3_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_bits); + + encode_mode3_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, pbits, cur_weights); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + inline int dist3(int lr, int lg, int lb, int hr, int hg, int hb) + { + return basisu::squarei(hr - lr) + basisu::squarei(hg - lg) + basisu::squarei(hb - lb); + } + + bool determine_3subsets(uint8_t* pFinal_3subsets, + const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b) + { + uint32_t subset_indices[16]; + int subset_means[2][3] = { }; + int subset_total[2] = { }; + + for (uint32_t i = 0; i < 16; i++) + { + const int rd = pPixels[i].r - block_mean_r; + const int gd = pPixels[i].g - block_mean_g; + const int bd = pPixels[i].b - block_mean_b; + + const uint32_t subset_index = ((float)rd * block_xr + (float)gd * block_xg + (float)bd * block_xb) > 0.0f; + + subset_indices[i] = subset_index; + + subset_means[subset_index][0] += pPixels[i].r; + subset_means[subset_index][1] += pPixels[i].g; + subset_means[subset_index][2] += pPixels[i].b; + + subset_total[subset_index]++; + } + + for (uint32_t i = 0; i < 2; i++) + { + const uint32_t t = subset_total[i]; + if (!t) + return false; + + subset_means[i][0] = (subset_means[i][0] + (t >> 1)) / t; + subset_means[i][1] = (subset_means[i][1] + (t >> 1)) / t; + subset_means[i][2] = (subset_means[i][2] + (t >> 1)) / t; + } + + int subset_sses[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const uint32_t subset_index = subset_indices[i]; + + subset_sses[subset_index] += dist3(pPixels[i].r, pPixels[i].g, pPixels[i].b, subset_means[subset_index][0], subset_means[subset_index][1], subset_means[subset_index][2]); + } + + const uint32_t subset_to_split = (subset_sses[1] > subset_sses[0]); + if (subset_total[subset_to_split] < 2) + return false; + + int lo_y = INT_MAX, hi_y = 0; + for (uint32_t i = 0; i < 16; i++) + { + if (subset_indices[i] != subset_to_split) + continue; + + int y = ((pPixels[i].r + pPixels[i].g + pPixels[i].b) << 4) + i; + + lo_y = basisu::minimum(lo_y, y); + hi_y = basisu::maximum(hi_y, y); + } + + const int lo_y_index = lo_y & 15, hi_y_index = hi_y & 15; + if (lo_y_index == hi_y_index) + return false; + + const int lr = pPixels[lo_y_index].r, lg = pPixels[lo_y_index].g, lb = pPixels[lo_y_index].b; + const int hr = pPixels[hi_y_index].r, hg = pPixels[hi_y_index].g, hb = pPixels[hi_y_index].b; + + memset(pFinal_3subsets, 2, 16); + + for (uint32_t i = 0; i < 16; i++) + { + if (subset_indices[i] == subset_to_split) + { + const int dist0 = dist3(lr, lg, lb, pPixels[i].r, pPixels[i].g, pPixels[i].b); + const int dist1 = dist3(hr, hg, hb, pPixels[i].r, pPixels[i].g, pPixels[i].b); + + pFinal_3subsets[i] = dist1 > dist0; + } + } + + return true; + } + + static inline int pop16(uint32_t x) + { +#if defined(_MSC_VER) + return __popcnt16((unsigned short)x); +#else + return __builtin_popcount(x & 0xFFFFu); +#endif + } + + int pick_3subset_pat_index(const uint8_t* pDesired_subsets, uint32_t& best_pat_index_first16) + { + best_pat_index_first16 = 0; + + uint16_t M[3]; + memset(M, 0, sizeof(M)); + + for (uint32_t i = 0; i < 16; i++) + { + uint32_t s = pDesired_subsets[i]; + M[s] |= (1 << i); + } + + const int n0 = pop16(M[0]), n1 = pop16(M[1]), n2 = 16 - n0 - n1; + + int best_score = -1; + int best_pat = 0; + + for (int p = 0; p < (int)MAX_PATTERNS3_TO_CHECK; ++p) + { + uint16_t S0 = (uint16_t)(g_part3_bitmasks[p] & 0xFFFFu); + uint16_t S1 = (uint16_t)(g_part3_bitmasks[p] >> 16); + + // Row sums for subsets 0 and 1 via 6 popcnts; derive subset 2 by subtraction + int C00 = pop16(M[0] & S0), C01 = pop16(M[0] & S1), C02 = n0 - C00 - C01; + int C10 = pop16(M[1] & S0), C11 = pop16(M[1] & S1), C12 = n1 - C10 - C11; + int C20 = pop16(M[2] & S0), C21 = pop16(M[2] & S1), C22 = n2 - C20 - C21; + + int s0 = C00 + C11 + C22; // (0,1,2) + int s1 = C00 + C12 + C21; // (0,2,1) + int s2 = C01 + C10 + C22; // (1,0,2) + int s3 = C01 + C12 + C20; // (1,2,0) + int s4 = C02 + C10 + C21; // (2,0,1) + int s5 = C02 + C11 + C20; // (2,1,0) + + // Argmax over 6 + int s = s0; + if (s1 > s) { s = s1; } + if (s2 > s) { s = s2; } + if (s3 > s) { s = s3; } + if (s4 > s) { s = s4; } + if (s5 > s) { s = s5; } + + if (s > best_score) + { + best_score = s; + best_pat = p; + + if (s == 16) + { + // perfect match so early out + if (p <= 15) + best_pat_index_first16 = best_pat; + break; + } + } + + if (p == 15) + { + // for mode 0 + best_pat_index_first16 = best_pat; + } + } + + return best_pat; + } + +#if 0 + static const uint8_t s_perms3[6][3] = { {0,1,2}, {0,2,1}, {1,0,2}, {1,2,0}, {2,0,1}, {2,1,0} }; + + int pick_3subset_pat_index_slow(const uint8_t* pDesired_subsets, uint32_t& best_pat_index_first16) + { + int best_pat = 0, best_dist = INT_MAX; + + for (uint32_t m = 0; m < 64; m++) + { + const uint8_t* pPat = &g_bc7_partition3[m * 16]; + + for (uint32_t p = 0; p < 6; p++) + { + int trial_dist = 0; + + for (uint32_t i = 0; i < 16; i++) + { + uint32_t s = s_perms3[p][pDesired_subsets[i]]; + + trial_dist += (s != pPat[i]); + + } // i + + if (trial_dist < best_dist) + { + best_dist = trial_dist; + best_pat = m; + } + + } // p + + if (m == 15) + best_pat_index_first16 = best_pat; + + } // m + + return best_pat; + } +#endif + + // false if packing failed (not enough unique colors) + bool pack_mode0_or_2_rgb(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, + int block_mean_r, int block_mean_g, int block_mean_b, float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { + (void)flags; + +#if BASISU_BC7F_PERF_STATS + g_total_mode02_evals++; +#endif + + uint8_t desired_3subsets[16]; + if (!determine_3subsets(desired_3subsets, pPixels, block_xr, block_xg, block_xb, block_mean_r, block_mean_g, block_mean_b)) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + if (pFinal_sse_est) + *pFinal_sse_est = 1e+9f; + + return false; + } + + uint32_t best_pat_indices[2]; // mode 0 and 2 + best_pat_indices[1] = pick_3subset_pat_index(desired_3subsets, best_pat_indices[0]); + + assert((best_pat_indices[0] <= 15) && (best_pat_indices[1] <= 63)); + + float total_quant_sse_mode[2] = { }; + float total_slam_to_line_sse_mode[2] = { }; + + int mode_total_c[2][3] = { }; + int mode_low_c[2][3] = { }, mode_high_c[2][3] = { }; + int mode_total_r[2][3] = { }, mode_total_g[2][3] = { }, mode_total_b[2][3] = { }; + + int spans[4] = { }; + + for (uint32_t mode_iter = 0; mode_iter < 2; mode_iter++) // mode 0 vs. mode 2 + { + if ((mode_iter) && (best_pat_indices[0] == best_pat_indices[1])) + { + for (uint32_t s = 0; s < 3; s++) + { + mode_total_c[1][s] = mode_total_c[0][s]; + + mode_low_c[1][s] = mode_low_c[0][s]; + mode_high_c[1][s] = mode_high_c[0][s]; + + mode_total_r[1][s] = mode_total_r[0][s]; + mode_total_g[1][s] = mode_total_g[0][s]; + mode_total_b[1][s] = mode_total_b[0][s]; + + total_slam_to_line_sse_mode[1] = total_slam_to_line_sse_mode[0]; + + } // subset + } + else + { + const uint32_t best_pat_index = best_pat_indices[mode_iter]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + int* pTotal_r = &mode_total_r[mode_iter][0]; + int* pTotal_g = &mode_total_g[mode_iter][0]; + int* pTotal_b = &mode_total_b[mode_iter][0]; + + int* pTotal_c = mode_total_c[mode_iter]; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int subset = pBest_pat[i]; + + pTotal_r[subset] += r; pTotal_g[subset] += g; pTotal_b[subset] += b; + pTotal_c[subset]++; + } + + int mean_r[3], mean_g[3], mean_b[3]; + for (uint32_t s = 0; s < 3; s++) + { + const uint32_t t = pTotal_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (pTotal_r[s] + h) / t; + mean_g[s] = (pTotal_g[s] + h) / t; + mean_b[s] = (pTotal_b[s] + h) / t; + } + + int icov[3][6] = { }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + + int r = (int)pPixels[i].r - mean_r[subset]; + int g = (int)pPixels[i].g - mean_g[subset]; + int b = (int)pPixels[i].b - mean_b[subset]; + icov[subset][0] += r * r; icov[subset][1] += r * g; icov[subset][2] += r * b; + icov[subset][3] += g * g; icov[subset][4] += g * b; + icov[subset][5] += b * b; + } + + int ar[3], ag[3], ab[3]; + + float total_slam_to_line_sse = 0.0f; + + for (uint32_t s = 0; s < 3; s++) + { + int block_max_var = basisu::maximum(icov[s][0], icov[s][3], icov[s][5]); + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[s][i]; + + const float sc = 1.0f / ((float)block_max_var + .0000125f); + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + total_slam_to_line_sse += estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + } // s + + total_slam_to_line_sse_mode[mode_iter] = total_slam_to_line_sse; + + int low_dot[3] = { INT_MAX, INT_MAX, INT_MAX }; + int high_dot[3] = { INT_MIN, INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset]; + + int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + for (uint32_t subset = 0; subset < 3; subset++) + { + mode_low_c[mode_iter][subset] = low_dot[subset] & 15; + mode_high_c[mode_iter][subset] = high_dot[subset] & 15; + } // subset + + } // if ((mode_iter) && (best_pat_indices[0] == best_pat_indices[1])) + + for (uint32_t subset = 0; subset < 3; subset++) + { + const uint32_t low_pixel = mode_low_c[mode_iter][subset]; + const uint32_t high_pixel = mode_high_c[mode_iter][subset]; + + for (uint32_t c = 0; c < 3; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + float subset_sse; + if (mode_iter == 0) + { + // mode 0: 4-bit endpoints, unique p-bits, 3-bit weights, slight p-bit endpoint scale factor + subset_sse = analytical_quant_est_sse(16, 8, 3, spans, nullptr, UNIQUE_PBIT_DISCOUNT, mode_total_c[mode_iter][subset]); + } + else + { + // mode 2: 5-bit endpoints, no p-bits, 2-bit weights, no endpoint scale factor + subset_sse = analytical_quant_est_sse(32, 4, 3, spans, nullptr, 1.0f, mode_total_c[mode_iter][subset]); + } + + total_quant_sse_mode[mode_iter] += subset_sse; + } // subset + + } // mode_iter + + const float total_sse_est_mode0 = total_quant_sse_mode[0] + total_slam_to_line_sse_mode[0]; + const float total_sse_est_mode2 = total_quant_sse_mode[1] + total_slam_to_line_sse_mode[1]; + + if (total_sse_est_mode0 < total_sse_est_mode2) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_sse_est_mode0; + + // Use mode 0 (high span) + if (total_sse_est_mode0 >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + return false; + } + + const uint32_t best_pat_index = best_pat_indices[0]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + const int* pLow_c = &mode_low_c[0][0]; + const int* pHigh_c = &mode_high_c[0][0]; + + const int* pTotal_r = &mode_total_r[0][0]; + const int* pTotal_g = &mode_total_g[0][0]; + const int* pTotal_b = &mode_total_b[0][0]; + + float xl[3][4], xh[3][4]; + + for (uint32_t s = 0; s < 3; s++) + { + const int lc = pLow_c[s]; + const int hc = pHigh_c[s]; + + xl[s][0] = (float)pPixels[lc].r * (1.0f / 255.0f); + xl[s][1] = (float)pPixels[lc].g * (1.0f / 255.0f); + xl[s][2] = (float)pPixels[lc].b * (1.0f / 255.0f); + xl[s][3] = 0.0f; + + xh[s][0] = (float)pPixels[hc].r * (1.0f / 255.0f); + xh[s][1] = (float)pPixels[hc].g * (1.0f / 255.0f); + xh[s][2] = (float)pPixels[hc].b * (1.0f / 255.0f); + xh[s][3] = 0.0f; + } // s + + uint32_t lr[3], lg[3], lb[3], hr[3], hg[3], hb[3], pbits[6]; + + for (uint32_t s = 0; s < 3; s++) + { + color_rgba el, eh; + determine_unique_pbits(3, 4, xl[s], xh[s], el, eh, &pbits[s << 1]); + + lr[s] = el[0]; lg[s] = el[1]; lb[s] = el[2]; + hr[s] = eh[0]; hg[s] = eh[1]; hb[s] = eh[2]; + + } // s + + uint8_t cur_weights[16]; + eval_weights_mode0_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + + float z00[3] = { 0.0f }, z10[3] = { 0.0f }, z11[3] = { 0.0f }; + float q00_r[3] = { 0.0f }; + float q00_g[3] = { 0.0f }; + float q00_b[3] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const uint32_t sel = cur_weights[i]; + assert(sel <= 7); + + z00[subset] += g_bc7_3bit_ls_tab[sel][0]; + z10[subset] += g_bc7_3bit_ls_tab[sel][1]; + z11[subset] += g_bc7_3bit_ls_tab[sel][2]; + + const float w = g_bc7_3bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 3; s++) + { + float q10_r = (float)pTotal_r[s] - q00_r[s]; + float q10_g = (float)pTotal_g[s] - q00_g[s]; + float q10_b = (float)pTotal_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float q = 1.0f / 255.0f; + + xl[s][0] = basisu::clamp(q * (iz10 * q00_r[s] + iz11 * q10_r), 0.0f, 1.0f); + xh[s][0] = basisu::clamp(q * (iz00 * q00_r[s] + iz01 * q10_r), 0.0f, 1.0f); + + xl[s][1] = basisu::clamp(q * (iz10 * q00_g[s] + iz11 * q10_g), 0.0f, 1.0f); + xh[s][1] = basisu::clamp(q * (iz00 * q00_g[s] + iz01 * q10_g), 0.0f, 1.0f); + + xl[s][2] = basisu::clamp(q * (iz10 * q00_b[s] + iz11 * q10_b), 0.0f, 1.0f); + xh[s][2] = basisu::clamp(q * (iz00 * q00_b[s] + iz01 * q10_b), 0.0f, 1.0f); + } // s + + for (uint32_t s = 0; s < 3; s++) + { + color_rgba el, eh; + determine_unique_pbits(3, 4, xl[s], xh[s], el, eh, &pbits[s << 1]); // fills in both pbit entries + + lr[s] = el[0]; lg[s] = el[1]; lb[s] = el[2]; + hr[s] = eh[0]; hg[s] = eh[1]; hb[s] = eh[2]; + + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode0_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + else + eval_weights_mode0_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, pbits, best_pat_index); + + encode_mode0_rgb_block(pBlock, best_pat_index, lr, lg, lb, hr, hg, hb, pbits, cur_weights); + } + else + { + if (pFinal_sse_est) + *pFinal_sse_est = total_sse_est_mode2; + + // Use mode 2 (low span) + if (total_sse_est_mode2 >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode02_bailouts++; +#endif + return false; + } + + const uint32_t best_pat_index = best_pat_indices[1]; + const uint8_t* pBest_pat = &g_bc7_partition3[best_pat_index * 16]; + + const int* pLow_c = &mode_low_c[1][0]; + const int* pHigh_c = &mode_high_c[1][0]; + + const int* pTotal_r = &mode_total_r[1][0]; + const int* pTotal_g = &mode_total_g[1][0]; + const int* pTotal_b = &mode_total_b[1][0]; + + uint32_t lr[3], lg[3], lb[3]; + uint32_t hr[3], hg[3], hb[3]; + + for (uint32_t s = 0; s < 3; s++) + { + const int lc = pLow_c[s]; + const int hc = pHigh_c[s]; + + lr[s] = to_5(pPixels[lc].r); + lg[s] = to_5(pPixels[lc].g); + lb[s] = to_5(pPixels[lc].b); + + hr[s] = to_5(pPixels[hc].r); + hg[s] = to_5(pPixels[hc].g); + hb[s] = to_5(pPixels[hc].b); + } + + uint8_t cur_weights[16]; + eval_weights_mode2_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + + float z00[3] = { 0.0f }, z10[3] = { 0.0f }, z11[3] = { 0.0f }; + float q00_r[3] = { 0.0f }; + float q00_g[3] = { 0.0f }; + float q00_b[3] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = pBest_pat[i]; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + } // i + + for (uint32_t s = 0; s < 3; s++) + { + float q10_r = (float)pTotal_r[s] - q00_r[s]; + float q10_g = (float)pTotal_g[s] - q00_g[s]; + float q10_b = (float)pTotal_b[s] - q00_b[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabs(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + hr[s] = to_5_clamp(iz00 * q00_r[s] + iz01 * q10_r); + lr[s] = to_5_clamp(iz10 * q00_r[s] + iz11 * q10_r); + + hg[s] = to_5_clamp(iz00 * q00_g[s] + iz01 * q10_g); + lg[s] = to_5_clamp(iz10 * q00_g[s] + iz11 * q10_g); + + hb[s] = to_5_clamp(iz00 * q00_b[s] + iz01 * q10_b); + lb[s] = to_5_clamp(iz10 * q00_b[s] + iz11 * q10_b); + } // s + + if (pActual_sse) + *pActual_sse = eval_weights_mode2_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + else + eval_weights_mode2_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, best_pat_index); + + encode_mode2_rgb_block(pBlock, best_pat_index, + lr, lg, lb, hr, hg, hb, cur_weights); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + bool pack_mode4_or_5(uint8_t* pBlock, const color_rgba* pOrig_pixels, uint32_t dp_chan_index, float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { + (void)flags; + +#if BASISU_BC7F_PERF_STATS + g_total_mode45_evals++; +#endif + + color_rgba pixels[16]; + const color_rgba* pPixels = pOrig_pixels; + + if (dp_chan_index != 3) + { + memcpy(pixels, pOrig_pixels, sizeof(color_rgba) * 16); + pPixels = pixels; + + for (uint32_t i = 0; i < 16; i++) + { + const uint8_t c = pixels[i][dp_chan_index]; + pixels[i][dp_chan_index] = pixels[i][3]; + pixels[i][3] = c; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r; + const int g = (int)pPixels[i].g - mean_g; + const int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + float cov3[6]; + for (uint32_t i = 0; i < 6; i++) + cov3[i] = (float)icov[i]; + + const int block_max_var3 = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + const float sc3 = block_max_var3 ? (1.0f / (float)block_max_var3) : 0; + const float wx3 = sc3 * cov3[0], wy3 = sc3 * cov3[3], wz3 = sc3 * cov3[5]; + + const float alt_xr = cov3[0] * wx3 + cov3[1] * wy3 + cov3[2] * wz3; + const float alt_xg = cov3[1] * wx3 + cov3[3] * wy3 + cov3[4] * wz3; + const float alt_xb = cov3[2] * wx3 + cov3[4] * wy3 + cov3[5] * wz3; + + // Same for mode 4/5 + const float rgb_slam_to_line_sse_est = estimate_slam_to_line_sse_3D(cov3, alt_xr, alt_xg, alt_xb); + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + const int rgb_spans[4] = { pPixels[high_c][0] - pPixels[low_c][0], pPixels[high_c][1] - pPixels[low_c][1], pPixels[high_c][2] - pPixels[low_c][2], 0 }; + const int a_span = max_a - min_a; + + const float SECOND_PLANE_SPAN_WEIGHT = (dp_chan_index == 3) ? 1.0f : 1.0f; + + const float mode_4_rgb_3bit_quant_sse_est = analytical_quant_est_sse(32, 8, 3, rgb_spans, nullptr, 1.0f, 16); // mode 4 rgb: 5-bit endpoints, using 3-bit weights for RGB + const float mode_4_a_2bit_quant_sse_est = analytical_quant_est_sse(64, 4, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 4 a: 6-bit endpoints, using 2-bit weights for RGB + + const float mode_4_rgb_2bit_quant_sse_est = analytical_quant_est_sse(32, 4, 3, rgb_spans, nullptr, 1.0f, 16); // mode 4 rgb: 5-bit endpoints, using 2-bit weights for RGB + const float mode_4_a_3bit_quant_sse_est = analytical_quant_est_sse(64, 8, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 4 a: 6-bit endpoints, using 3-bit weights for RGB + + const float total_mode_4_rgb3_a2_sse_est = rgb_slam_to_line_sse_est + mode_4_rgb_3bit_quant_sse_est + mode_4_a_2bit_quant_sse_est; + const float total_mode_4_rgb2_a3_sse_est = rgb_slam_to_line_sse_est + mode_4_rgb_2bit_quant_sse_est + mode_4_a_3bit_quant_sse_est; + + const float mode_5_rgb_quant_sse_est = analytical_quant_est_sse(128, 4, 3, rgb_spans, nullptr, 1.0f, 16); // mode 5 rgb: 7-bit endpoints, using 2-bit weights for RGB + const float mode_5_a_quant_sse_est = analytical_quant_est_sse(256, 4, a_span, SECOND_PLANE_SPAN_WEIGHT, 1.0f, 16); // mode 5 a: 8-bit endpoints, using 2-bit weights for RGB + const float total_mode_5_rgba_sse_est = rgb_slam_to_line_sse_est + mode_5_rgb_quant_sse_est + mode_5_a_quant_sse_est; + + if (total_mode_5_rgba_sse_est < basisu::minimum(total_mode_4_rgb3_a2_sse_est, total_mode_4_rgb2_a3_sse_est)) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_5_rgba_sse_est; + + // Mode 5 - low RGB/A span + if (total_mode_5_rgba_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_7(pPixels[low_c].r), lg = to_7(pPixels[low_c].g), lb = to_7(pPixels[low_c].b), la = min_a; + int hr = to_7(pPixels[high_c].r), hg = to_7(pPixels[high_c].g), hb = to_7(pPixels[high_c].b), ha = max_a; + + uint8_t cur_weights0[16]; // rgb 2-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode5_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode5_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 4, + g_bc7_2bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (127.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (127.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (127.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (127.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (127.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (127.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode5_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode5_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode5_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode5_2bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 4, + g_bc7_2bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal); + ha = fast_roundf_int(nah); + + if (pActual_sse) + a_sse = eval_weights_mode5_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode5_2bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode5_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3); + } + else if (total_mode_4_rgb3_a2_sse_est < total_mode_4_rgb2_a3_sse_est) + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_4_rgb3_a2_sse_est; + + // mode 4, rgb 3-bits, alpha 2-bits - high span RGB, low span in A, index bit=1 + if (total_mode_4_rgb3_a2_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_5(pPixels[low_c].r), lg = to_5(pPixels[low_c].g), lb = to_5(pPixels[low_c].b), la = to_6(min_a); + int hr = to_5(pPixels[high_c].r), hg = to_5(pPixels[high_c].g), hb = to_5(pPixels[high_c].b), ha = to_6(max_a); + + uint8_t cur_weights0[16]; // rgb 3-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode4_3bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_3bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 8, + g_bc7_3bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (31.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (31.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (31.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (31.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (31.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (31.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode4_3bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_3bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode4_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_2bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 4, + g_bc7_2bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal * (63.0f / 255.0f)); + ha = fast_roundf_int(nah * (63.0f / 255.0f)); + + if (pActual_sse) + a_sse = eval_weights_mode4_2bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_2bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode4_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3, 1); + } + else + { + if (pFinal_sse_est) + *pFinal_sse_est = total_mode_4_rgb2_a3_sse_est; + + // mode 4, rgb 2-bits, alpha 3-bits - low span RGB, high span in A, index bit=0 + if (total_mode_4_rgb2_a3_sse_est >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode45_bailouts++; +#endif + return false; + } + + int lr = to_5(pPixels[low_c].r), lg = to_5(pPixels[low_c].g), lb = to_5(pPixels[low_c].b), la = to_6(min_a); + int hr = to_5(pPixels[high_c].r), hg = to_5(pPixels[high_c].g), hb = to_5(pPixels[high_c].b), ha = to_6(max_a); + + uint8_t cur_weights0[16]; // rgb 2-bits + if (pActual_sse) + *pActual_sse = eval_weights_mode4_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights0, 4, + g_bc7_2bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + lr = fast_roundf_int(xl[0] * (31.0f / 255.0f)); + lg = fast_roundf_int(xl[1] * (31.0f / 255.0f)); + lb = fast_roundf_int(xl[2] * (31.0f / 255.0f)); + + hr = fast_roundf_int(xh[0] * (31.0f / 255.0f)); + hg = fast_roundf_int(xh[1] * (31.0f / 255.0f)); + hb = fast_roundf_int(xh[2] * (31.0f / 255.0f)); + + if (pActual_sse) + *pActual_sse = eval_weights_mode4_2bit_rgb_sse(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + else + eval_weights_mode4_2bit_rgb(pPixels, cur_weights0, lr, lg, lb, hr, hg, hb); + } + + uint8_t cur_weights1[16]; // alpha 2-bits + uint32_t a_sse = 0; + if (pActual_sse) + a_sse = eval_weights_mode4_3bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_3bit_a(pPixels, cur_weights1, la, ha); + + float nal, nah; + if (compute_least_squares_endpoints_1D( + 16, cur_weights1, 8, + g_bc7_3bit_ls_tab, + nal, nah, + pPixels, 3, + (float)total_a)) + { + la = fast_roundf_int(nal * (63.0f / 255.0f)); + ha = fast_roundf_int(nah * (63.0f / 255.0f)); + + if (pActual_sse) + a_sse = eval_weights_mode4_3bit_a_sse(pPixels, cur_weights1, la, ha); + else + eval_weights_mode4_3bit_a(pPixels, cur_weights1, la, ha); + } + + if (pActual_sse) + *pActual_sse += a_sse; + + encode_mode4_rgba_block(pBlock, + lr, lg, lb, la, + hr, hg, hb, ha, + cur_weights0, cur_weights1, (dp_chan_index + 1) & 3, 0); + } + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pOrig_pixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + bool pack_mode7_rgba(uint8_t* pBlock, const color_rgba* pPixels, + float block_xr, float block_xg, float block_xb, float block_xa, + int block_mean_r, int block_mean_g, int block_mean_b, int block_mean_a, + float sse_est_to_beat, uint32_t flags, + float* pFinal_sse_est = nullptr, + uint32_t* pActual_sse = nullptr) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode7_evals++; +#endif + + uint32_t desired_pat_bits = 0; + + for (uint32_t i = 0; i < 16; i++) + { + const float r = (float)(pPixels[i].r - block_mean_r); + const float g = (float)(pPixels[i].g - block_mean_g); + const float b = (float)(pPixels[i].b - block_mean_b); + const float a = (float)(pPixels[i].a - block_mean_a); + + const uint32_t subset = (r * block_xr + g * block_xg + b * block_xb + a * block_xa) > 0.0f; + + desired_pat_bits |= (subset << i); + } + + uint32_t best_diff = UINT32_MAX; + for (uint32_t p = 0; p < MAX_PATTERNS2_TO_CHECK; p++) + { + const uint32_t bc6h_pat_bits = g_bc7_part2_bitmasks[p]; + + int diff = popcount32(bc6h_pat_bits ^ desired_pat_bits); + int diff_inv = 16 - diff; + + uint32_t min_diff = (basisu::minimum(diff, diff_inv) << 8) | p; + if (min_diff < best_diff) + best_diff = min_diff; + } // p + + const uint32_t best_pat_index = best_diff & 0xFF; + const uint32_t best_pat_bits = g_bc7_part2_bitmasks[best_pat_index]; + + int total_r[2] = { }, total_g[2] = { }, total_b[2] = { }, total_a[2] = { }, total_c[2] = { }; + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + const int subset = (best_pat_bits >> i) & 1; + + total_r[subset] += r; total_g[subset] += g; total_b[subset] += b; total_a[subset] += a; + total_c[subset]++; + } + + int mean_r[2], mean_g[2], mean_b[2], mean_a[2]; + for (uint32_t s = 0; s < 2; s++) + { + const uint32_t t = total_c[s]; + const uint32_t h = (t >> 1); + + mean_r[s] = (total_r[s] + h) / t; + mean_g[s] = (total_g[s] + h) / t; + mean_b[s] = (total_b[s] + h) / t; + mean_a[s] = (total_a[s] + h) / t; + } + + int icov4[2][10] = { { }, { } }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int s = (best_pat_bits >> i) & 1; + + int r = (int)pPixels[i].r - mean_r[s]; + int g = (int)pPixels[i].g - mean_g[s]; + int b = (int)pPixels[i].b - mean_b[s]; + int a = (int)pPixels[i].a - mean_a[s]; + + icov4[s][0] += r * r; icov4[s][1] += r * g; icov4[s][2] += r * b; icov4[s][3] += r * a; + icov4[s][4] += g * g; icov4[s][5] += g * b; icov4[s][6] += g * a; + icov4[s][7] += b * b; icov4[s][8] += b * a; + icov4[s][9] += a * a; + } + + int ar[2], ag[2], ab[2], aa[2]; + + float slam_to_line_sse_est = 0.0f; + + for (uint32_t s = 0; s < 2; s++) + { + const int block_max_var4 = basisu::maximum(icov4[s][0], icov4[s][4], icov4[s][7], icov4[s][9]); + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[s][i]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + const float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + const float x0 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + const float y0 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + const float z0 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + const float w0 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + const float x1 = cov4[0] * x0 + cov4[1] * y0 + cov4[2] * z0 + cov4[3] * w0; + const float y1 = cov4[1] * x0 + cov4[4] * y0 + cov4[5] * z0 + cov4[6] * w0; + const float z1 = cov4[2] * x0 + cov4[5] * y0 + cov4[7] * z0 + cov4[8] * w0; + const float w1 = cov4[3] * x0 + cov4[6] * y0 + cov4[8] * z0 + cov4[9] * w0; + + slam_to_line_sse_est += estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1); + + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabsf(w1)); + if (fabsf(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + ar[s] = (int)((uint32_t)saxis_r << 4U); + ag[s] = (int)((uint32_t)saxis_g << 4U); + ab[s] = (int)((uint32_t)saxis_b << 4U); + aa[s] = (int)((uint32_t)saxis_a << 4U); + } // s + + int low_dot[2] = { INT_MAX, INT_MAX }; + int high_dot[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const int saxis_r = ar[subset], saxis_g = ag[subset], saxis_b = ab[subset], saxis_a = aa[subset]; + + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + const int dot = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + + low_dot[subset] = basisu::minimum(low_dot[subset], dot); + high_dot[subset] = basisu::maximum(high_dot[subset], dot); + } + + int low_c[2] = { low_dot[0] & 15, low_dot[1] & 15 }; + int high_c[2] = { high_dot[0] & 15, high_dot[1] & 15 }; + + float quant_err_sse_est = 0; + + for (uint32_t subset = 0; subset < 2; subset++) + { + const uint32_t low_pixel = low_c[subset]; + const uint32_t high_pixel = high_c[subset]; + + int spans[4]; + for (uint32_t c = 0; c < 4; c++) + spans[c] = pPixels[high_pixel][c] - pPixels[low_pixel][c]; + + // mode 7: 5-bit endpoints, unique pbits, 2 bit weights, 4 chans + quant_err_sse_est += analytical_quant_est_sse(32, 4, 4, spans, nullptr, (flags & cPackBC7FlagPBitOpt) ? UNIQUE_PBIT_DISCOUNT : 1.0f, total_c[subset]); + + } // subset + + const float total_mode7_est_sse = slam_to_line_sse_est + quant_err_sse_est; + + if (pFinal_sse_est) + *pFinal_sse_est = total_mode7_est_sse; + + if (total_mode7_est_sse >= sse_est_to_beat) + { +#if BASISU_BC7F_PERF_STATS + g_total_mode7_bailouts++; +#endif + return false; + } + + uint32_t lr[2], lg[2], lb[2], la[2]; + uint32_t hr[2], hg[2], hb[2], ha[2]; + uint32_t pbits[4]; + + for (uint32_t s = 0; s < 2; s++) + { + const int lc = low_c[s], hc = high_c[s]; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[lc].r * q, (float)pPixels[lc].g * q, (float)pPixels[lc].b * q, (float)pPixels[lc].a * q }; + float sxh[4] = { (float)pPixels[hc].r * q, (float)pPixels[hc].g * q, (float)pPixels[hc].b * q, (float)pPixels[hc].a * q }; + + color_rgba bestMinColor, bestMaxColor; + determine_unique_pbits(4, 5, sxl, sxh, bestMinColor, bestMaxColor, &pbits[s * 2]); + + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; la[s] = bestMinColor.a; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; ha[s] = bestMaxColor.a; + } + else + { + const uint32_t l_pbit = (pPixels[lc].a >= 129); + const uint32_t h_pbit = (pPixels[hc].a >= 129); + + pbits[s * 2 + 0] = l_pbit; + pbits[s * 2 + 1] = h_pbit; + + lr[s] = to_5(pPixels[lc].r, l_pbit); + lg[s] = to_5(pPixels[lc].g, l_pbit); + lb[s] = to_5(pPixels[lc].b, l_pbit); + la[s] = to_5(pPixels[lc].a, l_pbit); + + hr[s] = to_5(pPixels[hc].r, h_pbit); + hg[s] = to_5(pPixels[hc].g, h_pbit); + hb[s] = to_5(pPixels[hc].b, h_pbit); + ha[s] = to_5(pPixels[hc].a, h_pbit); + } + } // s + + uint8_t cur_weights[16]; + + eval_weights_mode7_rgba(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + + float z00[2] = { 0.0f }, z10[2] = { 0.0f }, z11[2] = { 0.0f }; + float q00_r[2] = { 0.0f }; + float q00_g[2] = { 0.0f }; + float q00_b[2] = { 0.0f }; + float q00_a[2] = { 0.0f }; + + for (uint32_t i = 0; i < 16; i++) + { + const int subset = (best_pat_bits >> i) & 1; + const uint32_t sel = cur_weights[i]; + assert(sel <= 3); + + z00[subset] += g_bc7_2bit_ls_tab[sel][0]; + z10[subset] += g_bc7_2bit_ls_tab[sel][1]; + z11[subset] += g_bc7_2bit_ls_tab[sel][2]; + + const float w = g_bc7_2bit_ls_tab[sel][3]; + + q00_r[subset] += w * (float)pPixels[i][0]; + q00_g[subset] += w * (float)pPixels[i][1]; + q00_b[subset] += w * (float)pPixels[i][2]; + q00_a[subset] += w * (float)pPixels[i][3]; + } // i + + for (uint32_t s = 0; s < 2; s++) + { + float q10_r = (float)total_r[s] - q00_r[s]; + float q10_g = (float)total_g[s] - q00_g[s]; + float q10_b = (float)total_b[s] - q00_b[s]; + float q10_a = (float)total_a[s] - q00_a[s]; + + float z01 = z10[s]; + + float det = z00[s] * z11[s] - z01 * z10[s]; + if (fabsf(det) < 1e-8f) + continue; + + det = 1.0f / det; + + float iz00, iz01, iz10, iz11; + iz00 = z11[s] * det; + iz01 = -z01 * det; + iz10 = -z10[s] * det; + iz11 = z00[s] * det; + + const float slr = iz10 * q00_r[s] + iz11 * q10_r; + const float shr = iz00 * q00_r[s] + iz01 * q10_r; + + const float slg = iz10 * q00_g[s] + iz11 * q10_g; + const float shg = iz00 * q00_g[s] + iz01 * q10_g; + + const float slb = iz10 * q00_b[s] + iz11 * q10_b; + const float shb = iz00 * q00_b[s] + iz01 * q10_b; + + const float sla = iz10 * q00_a[s] + iz11 * q10_a; + const float sha = iz00 * q00_a[s] + iz01 * q10_a; + + if (flags & cPackBC7FlagPBitOpt) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { basisu::clamp(slr * q, 0.0f, 1.0f), basisu::clamp(slg * q, 0.0f, 1.0f), basisu::clamp(slb * q, 0.0f, 1.0f), basisu::clamp(sla * q, 0.0f, 1.0f) }; + float sxh[4] = { basisu::clamp(shr * q, 0.0f, 1.0f), basisu::clamp(shg * q, 0.0f, 1.0f), basisu::clamp(shb * q, 0.0f, 1.0f), basisu::clamp(sha * q, 0.0f, 1.0f) }; + + color_rgba bestMinColor, bestMaxColor; + determine_unique_pbits(4, 5, sxl, sxh, bestMinColor, bestMaxColor, &pbits[s * 2]); + + lr[s] = bestMinColor.r, lg[s] = bestMinColor.g, lb[s] = bestMinColor.b; la[s] = bestMinColor.a; + hr[s] = bestMaxColor.r, hg[s] = bestMaxColor.g, hb[s] = bestMaxColor.b; ha[s] = bestMaxColor.a; + } + else + { + const uint32_t l_pbit = (sla >= 129.0f); + const uint32_t h_pbit = (sha >= 129.0f); + + pbits[s * 2 + 0] = l_pbit; + pbits[s * 2 + 1] = h_pbit; + + lr[s] = to_5_clamp(slr, l_pbit); + lg[s] = to_5_clamp(slg, l_pbit); + lb[s] = to_5_clamp(slb, l_pbit); + la[s] = to_5_clamp(sla, l_pbit); + + hr[s] = to_5_clamp(shr, h_pbit); + hg[s] = to_5_clamp(shg, h_pbit); + hb[s] = to_5_clamp(shb, h_pbit); + ha[s] = to_5_clamp(sha, h_pbit); + } + + } // s + + if (pActual_sse) + { + *pActual_sse = eval_weights_mode7_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + } + else + { + eval_weights_mode7_rgba(pPixels, cur_weights, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, best_pat_bits); + } + + encode_mode7_rgba_block(pBlock, best_pat_index, + lr, lg, lb, la, + hr, hg, hb, ha, + pbits, cur_weights); + +#ifdef _DEBUG + if (pActual_sse) + { + const uint32_t expected_sse = calc_sse(pBlock, pPixels); + assert(expected_sse == *pActual_sse); + } +#endif + + return true; + } + + const int TRIVIAL_BLOCK_THRESH_RGB = 20 * 16; // skip PCA/LS threshold (uses trivial mode 6 encoder) + const int TRIVIAL_BLOCK_THRESH_RGBA = 2 * 16; + + // dual plane +#if 0 + const int DP_BLOCK_VAR_THRESH = 1 * 16; // use dual plane threshold + const float STRONG_CORR_THRESH = .98f; +#else + const int DP_BLOCK_VAR_THRESH = 2 * 16; // use dual plane threshold + const float STRONG_CORR_THRESH = .85f; +#endif + + // 2-3 subsets +#if 0 + const float HIGH_ORTHO_ENERGY_THRESH = 1.0f * 16.0f; // use 2+ subsets threshold + const int MIN_BLOCK_MAX_VAR_23SUBSETS = 4 * 16; + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH = .004f; +#else + const int MIN_BLOCK_MAX_VAR_23SUBSETS = 100 * 16; + const float HIGH_ORTHO_ENERGY_THRESH = 1.0f * 16.0f; // use 2+ subsets threshold + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH = .004f; +#endif + +#if 0 + const int DP_BLOCK_VAR_THRESH_RGBA = 2 * 16; // use dual plane threshold + const float ALPHA_DECORR_THRESHOLD = .9f; + const float STRONG_DECORR_THRESH_RGBA = .85f; +#else + const int DP_BLOCK_VAR_THRESH_RGBA = 1 * 16; // use dual plane threshold + //const float ALPHA_DECORR_THRESHOLD = .98f; + const float ALPHA_DECORR_THRESHOLD = .995f; + const float STRONG_DECORR_THRESH_RGBA = .85f; +#endif + + // 3 subsets + const int MIN_BLOCK_MAX_VAR_3SUBSETS = 500 * 16; // use 3 subsets threshold + + //------------------------------------------------------------------------------------------------------- + + // Note: solid block check assumes A's all == 255. + void fast_pack_bc7_rgb_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgb_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + } + + int total_r = 0, total_g = 0, total_b = 0; + + int min_r = 255, min_g = 255, min_b = 255; + int max_r = 0, max_g = 0, max_b = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + + total_r += r; total_g += g; total_b += b; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int r = (int)pPixels[i].r - mean_r; + int g = (int)pPixels[i].g - mean_g; + int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + // not redundant due to uint32_t test above, which could be fooled by alpha accidentally passed in + if (!block_max_var) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + if ((flags & cPackBC7FlagUseDualPlaneRGB) && (block_max_var >= DP_BLOCK_VAR_THRESH)) + { + // 0,1 + // 0,2 + // 1,2 + const bool has_r = icov[0] > 16, has_g = icov[3] > 16, has_b = icov[5] > 16; + + const uint32_t total_active_chans = has_r + has_g + has_b; + + if (total_active_chans >= 2) + { + const float r_var = (float)icov[0], g_var = (float)icov[3], b_var = (float)icov[5]; + + const float rg_corr = (has_r && has_g) ? fabs((float)icov[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov[4] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + if (min_p < STRONG_CORR_THRESH) + { + if (total_active_chans == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var < TRIVIAL_BLOCK_THRESH_RGB))) + { + //pack_mode5_solid(pBlock, color_rgba(0, 255, 0, 255)); + //return; + + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b) + i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + // An alternative would be to set A's=1.0 here and bias the p-bit optimizer to lower A RMSE. + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; + p1 = 1; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + return; + } + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[i]; + + const float sc = block_max_var ? (1.0f / (float)block_max_var) : 0; + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + // quite rough mode 6 SSE estimate (explictly higher bound): if some other mode can't even beat this, don't use it and we fall back to a decently strong mode 6 + const int spans[4] = { max_r - min_r, max_g - min_g, max_b - min_b, 0 }; + + // need_sse_estimates MUST be set correctly or subtle mode selection issues will occur. + const bool need_sse_estimates = ((flags & cPackBC7FlagUse2SubsetsRGB) != 0) || (desired_dp_chan >= 0); + + float mode6_ortho_ratio = 0; + const float mode6_slam_to_line_sse_est = need_sse_estimates ? estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb, &mode6_ortho_ratio) : 0; + const float mode6_sse_est = need_sse_estimates ? (mode6_slam_to_line_sse_est + analytical_quant_est_sse(128, 16, 3, spans, nullptr, 1.0f, 16)) : 0; + + // Prefer 2/3-subsets over dual plane + // TODO: Use mode 6 sse est? + if ((flags & cPackBC7FlagUse2SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_23SUBSETS) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH)) + { + assert(need_sse_estimates); + + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + //pack_mode5_solid(pBlock, color_rgba(255, 255, 0, 255)); + //return; + + if ((flags & cPackBC7FlagUse3SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_3SUBSETS)) + { + //pack_mode5_solid(pBlock, color_rgba(255, 0, 255, 255)); + //return; + +#if 0 + if (pack_mode0_or_2_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags)) + { + return; + } +#else + float mode0_or_2_sse_est = 1e+9f; + if (pack_mode0_or_2_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags, &mode0_or_2_sse_est)) + { + float mode1_or_3_sse_est = 1e+9f; + + uint8_t temp_2subset_block[sizeof(basist::bc7_block)]; + if (pack_mode1_or_3_rgb(temp_2subset_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode0_or_2_sse_est, flags, &mode1_or_3_sse_est)) + { + assert(mode1_or_3_sse_est < mode0_or_2_sse_est); + memcpy(pBlock, temp_2subset_block, sizeof(basist::bc7_block)); + } + + return; + } +#endif + } + + if (pack_mode1_or_3_rgb(pBlock, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, mode6_sse_est, flags)) + return; + } + } + + // Use dual plane over mode 6 + if (desired_dp_chan >= 0) + { + assert(need_sse_estimates); + + if (pack_mode4_or_5(pBlock, pPixels, desired_dp_chan, mode6_sse_est, flags)) + return; + + } // if (desired_dp_chan >= 0) + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + +#if BASISU_BC7F_USE_SSE41 + int low_c, high_c; + bc7_proj_minmax_indices_sse41(pPixels, saxis_r, saxis_g, saxis_b, &low_c, &high_c); +#else + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + int low_c = low_dot & 15; + int high_c = high_dot & 15; +#endif + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + // explictly force pbits to 1, that way alpha is always 255 and we don't slow down the entire encoder by 4-8% for a tiny ~.1 dB PSNR gain (not worth it) + p0 = 1, p1 = 1; + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, 0.0f }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, 0.0f }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits( + 3, 7, sxl, sxh, + bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; p1 = 1; + lr = to_7(xl[0], p0); + lg = to_7(xl[1], p0); + lb = to_7(xl[2], p0); + + hr = to_7(xh[0], p1); + hg = to_7(xh[1], p1); + hb = to_7(xh[2], p1); + } + +#if BASISU_BC7F_USE_SSE41 + eval_weights_mode6_rgb_sse41(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#else + eval_weights_mode6_rgb(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); +#endif + } + + //pack_mode5_solid(pBlock, color_rgba(0, 0, 255, 255)); + //return; + + // pbits set to 1 to ensure alpha is always decoded to fully opaque (255) + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + } + + //------------------------------------------------------------------------------------------------------- + const int MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA = 100 * 16; + const float HIGH_ORTHO_ENERGY_THRESH_RGBA = 1.0f * 16.0f; // use 2+ subsets threshold + const float ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA = .004f; + + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgb_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + } + + int total_r = 0, total_g = 0, total_b = 0; + + int min_r = 255, min_g = 255, min_b = 255; + int max_r = 0, max_g = 0, max_b = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + + total_r += r; total_g += g; total_b += b; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); + } + + int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4; + + // covar rows are: + // 0, 1, 2 + // 1, 3, 4 + // 2, 4, 5 + int icov[6] = { 0, 0, 0, 0, 0, 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int r = (int)pPixels[i].r - mean_r; + int g = (int)pPixels[i].g - mean_g; + int b = (int)pPixels[i].b - mean_b; + icov[0] += r * r; icov[1] += r * g; icov[2] += r * b; + icov[3] += g * g; icov[4] += g * b; + icov[5] += b * b; + } + + int block_max_var = basisu::maximum(icov[0], icov[3], icov[5]); // not divided by 16, i.e. scaled by 16 + + // not redundant due to uint32_t test above, which could be fooled by alpha accidentally passed in + if (!block_max_var) + { +#if BASISU_BC7F_PERF_STATS + g_total_solid_blocks++; +#endif + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + const bool non_analytical_flag = (flags & cPackBC7FlagNonAnalyticalRGB) != 0; + + if ((flags & cPackBC7FlagUseDualPlaneRGB) && + ((!non_analytical_flag && (block_max_var >= DP_BLOCK_VAR_THRESH)) || (non_analytical_flag && (block_max_var >= 16)))) + { + // 0,1 + // 0,2 + // 1,2 + const bool has_r = icov[0] > 16, has_g = icov[3] > 16, has_b = icov[5] > 16; + + const uint32_t total_active_chans = has_r + has_g + has_b; + + if (total_active_chans >= 2) + { + const float r_var = (float)icov[0], g_var = (float)icov[3], b_var = (float)icov[5]; + + const float rg_corr = (has_r && has_g) ? fabs((float)icov[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov[4] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + + const float corr_thresh = non_analytical_flag ? .999f : STRONG_CORR_THRESH; + + if (min_p < corr_thresh) + { + if (total_active_chans == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var < TRIVIAL_BLOCK_THRESH_RGB))) + { + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b) + i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + // An alternative would be to set A's=1.0 here and bias the p-bit optimizer to lower A RMSE. + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + p0 = 1; + p1 = 1; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == mode6_actual_sse); + } +#endif + + return mode6_actual_sse; + } + + float cov[6]; + for (uint32_t i = 0; i < 6; i++) + cov[i] = (float)icov[i]; + + const float sc = block_max_var ? (1.0f / (float)block_max_var) : 0; + const float wx = sc * cov[0], wy = sc * cov[3], wz = sc * cov[5]; + + const float alt_xr = cov[0] * wx + cov[1] * wy + cov[2] * wz; + const float alt_xg = cov[1] * wx + cov[3] * wy + cov[4] * wz; + const float alt_xb = cov[2] * wx + cov[4] * wy + cov[5] * wz; + + int saxis_r = 306, saxis_g = 601, saxis_b = 117; + + float k = basisu::maximum(fabsf(alt_xr), fabsf(alt_xg), fabsf(alt_xb)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(alt_xr * m); + saxis_g = (int)(alt_xg * m); + saxis_b = (int)(alt_xb * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + +#if BASISU_BC7F_USE_SSE41 + int low_c, high_c; + bc7_proj_minmax_indices_sse41(pPixels, saxis_r, saxis_g, saxis_b, &low_c, &high_c); +#else + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + int low_c = low_dot & 15; + int high_c = high_dot & 15; +#endif + + int p0, p1, lr, lg, lb, hr, hg, hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, 0 }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, 0 }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(3, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b; + } + else + { + // explictly force pbits to 1, that way alpha is always 255 and we don't slow down the entire encoder by 4-8% for a tiny ~.1 dB PSNR gain (not worth it) + p0 = 1, p1 = 1; + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1); + } + + uint8_t cur_weights[16]; + + uint32_t mode6_actual_sse = eval_weights_mode6_rgb_sse(pPixels, cur_weights, lr, lg, lb, hr, hg, hb, p0, p1); + + if (mode6_actual_sse) + { + vec4F xl, xh; + bool res = compute_least_squares_endpoints_3D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b); + + if (res) + { + int trial_p0, trial_p1, trial_lr, trial_lg, trial_lb, trial_hr, trial_hg, trial_hb; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, 0.0f }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, 0.0f }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits( + 3, 7, sxl, sxh, + bestMinColor, bestMaxColor, best_pbits); + + trial_p0 = best_pbits[0], trial_p1 = best_pbits[1]; + trial_lr = bestMinColor.r, trial_lg = bestMinColor.g, trial_lb = bestMinColor.b; + trial_hr = bestMaxColor.r, trial_hg = bestMaxColor.g, trial_hb = bestMaxColor.b; + } + else + { + trial_p0 = 1; trial_p1 = 1; + trial_lr = to_7(xl[0], trial_p0); + trial_lg = to_7(xl[1], trial_p0); + trial_lb = to_7(xl[2], trial_p0); + + trial_hr = to_7(xh[0], trial_p1); + trial_hg = to_7(xh[1], trial_p1); + trial_hb = to_7(xh[2], trial_p1); + } + + uint8_t trial_weights[16]; + uint32_t mode6_ls_actual_sse = eval_weights_mode6_rgb_sse(pPixels, trial_weights, trial_lr, trial_lg, trial_lb, trial_hr, trial_hg, trial_hb, trial_p0, trial_p1); + if (mode6_ls_actual_sse < mode6_actual_sse) + { + mode6_actual_sse = mode6_ls_actual_sse; + memcpy(cur_weights, trial_weights, 16); + p0 = trial_p0; p1 = trial_p1; + lr = trial_lr; lg = trial_lg; lb = trial_lb; + hr = trial_hr; hg = trial_hg; hb = trial_hb; + } + } + } + + uint32_t mode02_actual_sse = UINT32_MAX; + uint8_t mode02_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode13_actual_sse = UINT32_MAX; + uint8_t mode13_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode45_actual_sse = UINT32_MAX; + uint8_t mode45_candidate_block[sizeof(basist::bc7_block)]; + + if (mode6_actual_sse) + { + if (non_analytical_flag) + { + // No gates: very expensive. + if (flags & cPackBC7FlagUse2SubsetsRGB) + { + if (flags & cPackBC7FlagUse3SubsetsRGB) + { + pack_mode0_or_2_rgb(mode02_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode02_actual_sse); + } + + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + + if (flags & cPackBC7FlagUseDualPlaneRGB) + pack_mode4_or_5(mode45_candidate_block, pPixels, (desired_dp_chan >= 0) ? desired_dp_chan : 1, 1e+9f, flags, nullptr, &mode45_actual_sse); // todo: determine best def channel here + } + else + { + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_3D(cov, alt_xr, alt_xg, alt_xb, &mode6_ortho_ratio); + + if ((flags & cPackBC7FlagUse2SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_23SUBSETS) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + + if ((flags & cPackBC7FlagUse3SubsetsRGB) && (block_max_var >= MIN_BLOCK_MAX_VAR_3SUBSETS)) + { + pack_mode0_or_2_rgb(mode02_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode02_actual_sse); + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + else + { + pack_mode1_or_3_rgb(mode13_candidate_block, pPixels, alt_xr, alt_xg, alt_xb, mean_r, mean_g, mean_b, 1e+9f, flags, nullptr, &mode13_actual_sse); + } + } + } + + if (desired_dp_chan >= 0) + { + assert(flags & cPackBC7FlagUseDualPlaneRGB); + + pack_mode4_or_5(mode45_candidate_block, pPixels, desired_dp_chan, 1e+9f, flags, nullptr, &mode45_actual_sse); + + } // if (desired_dp_chan >= 0) + } + } + + const uint32_t best_actual_sse = basisu::minimum(mode6_actual_sse, mode02_actual_sse, mode13_actual_sse, mode45_actual_sse); + + if ((mode45_actual_sse != UINT32_MAX) && (best_actual_sse == mode45_actual_sse)) + { + memcpy(pBlock, mode45_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode02_actual_sse != UINT32_MAX) && (best_actual_sse == mode02_actual_sse)) + { + memcpy(pBlock, mode02_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode13_actual_sse != UINT32_MAX) && (best_actual_sse == mode13_actual_sse)) + { + memcpy(pBlock, mode13_candidate_block, sizeof(basist::bc7_block)); + } + else + { + assert(mode6_actual_sse == best_actual_sse); + + // pbits set to 1 to ensure alpha is always decoded to fully opaque (255) + encode_mode6_rgba_block(pBlock, + lr, lg, lb, 127, p0, + hr, hg, hb, 127, p1, + cur_weights); + } + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == best_actual_sse); + } +#endif + + return best_actual_sse; + } + + //------------------------------------------------------------------------------------------------------- + + void fast_pack_bc7_rgba_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgba_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + pack_mode5_solid(pBlock, pPixels[0]); + return; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + assert((min_r != max_r) || (min_g != max_g) || (min_b != max_b) || (min_a != max_a)); + + const int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4, mean_a = (total_a + 8) >> 4; + + // covar rows are: + int icov4[10] = { }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r, g = (int)pPixels[i].g - mean_g, b = (int)pPixels[i].b - mean_b, a = (int)pPixels[i].a - mean_a; + + icov4[0] += r * r; icov4[1] += r * g; icov4[2] += r * b; icov4[3] += r * a; + icov4[4] += g * g; icov4[5] += g * b; icov4[6] += g * a; + icov4[7] += b * b; icov4[8] += b * a; + icov4[9] += a * a; + } + + const int block_max_var4 = basisu::maximum(icov4[0], icov4[4], icov4[7], icov4[9]); // not divided by 16, i.e. scaled by 16 + assert(block_max_var4); // solid blocks already filtered out + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + if ((flags & cPackBC7FlagUseDualPlaneRGBA) && (block_max_var4 >= DP_BLOCK_VAR_THRESH_RGBA)) + { + // Prefer A, if not strongly decorrelated then check RGB. + const float r_var = (float)icov4[0], g_var = (float)icov4[4], b_var = (float)icov4[7], a_var = (float)icov4[9]; + + const bool has_a = icov4[9] > 0; + + if (has_a) + { + const float p_03 = icov4[0] ? fabs((float)icov4[3] / sqrtf(r_var * a_var)) : 1.0f; + const float p_13 = icov4[4] ? fabs((float)icov4[6] / sqrtf(g_var * a_var)) : 1.0f; + const float p_23 = icov4[7] ? fabs((float)icov4[8] / sqrtf(b_var * a_var)) : 1.0f; + + const float min_p = basisu::minimum(p_03, p_13, p_23); + if (min_p < ALPHA_DECORR_THRESHOLD) + { + desired_dp_chan = 3; +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_a++; +#endif + } + } + + if (desired_dp_chan < 0) + { + const bool has_r = icov4[0] > 16, has_g = icov4[4] > 16, has_b = icov4[7] > 16; + const uint32_t total_active_chans_rgb = has_r + has_g + has_b; + + if (total_active_chans_rgb >= 2) + { + const float rg_corr = (has_r && has_g) ? fabs((float)icov4[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov4[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov4[5] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + if (min_p < STRONG_DECORR_THRESH_RGBA) + { + if (total_active_chans_rgb == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var4 < TRIVIAL_BLOCK_THRESH_RGBA))) + { + //pack_mode5_solid(pBlock, color_rgba(0, 255, 0, 255)); + //return; + + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b + (16 * 4) * pPixels[i].a); + assert((y & 0xF) == 0); + y += i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + return; + } + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[i]; + + // all channel pairs: + // 0,1=1 + // 0,2=2 + // 0,3=3 + // 1,2=5 + // 1,3=6 + // 2,3=8 + //const float r_var = cov4[0], g_var = cov4[4], b_var = cov4[7], a_var = cov4[9]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // TODO + float x1, y1, z1, w1; + for (uint32_t i = 0; i < 4; i++) + { + x1 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + y1 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + z1 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + w1 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + float t = sqrtf(x1 * x1 + y1 * y1 + z1 * z1 + w1 * w1); + if (t > basisu::SMALL_FLOAT_VAL) + { + t = 1.0f / t; + x1 *= t; y1 *= t; z1 *= t; w1 *= t; + } + else + { + x1 = y1 = z1 = w1 = .25f; + } + + wx = x1; wy = y1; wz = z1; wa = w1; + } + + const int spans[4] = { max_r - min_r, max_g - min_g, max_b - min_b, max_a - min_a }; + + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1, &mode6_ortho_ratio); + const float mode6_sse_est = (mode6_slam_to_line_sse_est + analytical_quant_est_sse(128, 16, 4, spans, nullptr, 1.0f, 16)); + + float mode45_sse_est = 1e+9f, mode7_sse_est = 1e+9f; + + uint8_t mode45_block[sizeof(basist::bc7_block)]; + const bool mode45_valid_flag = (desired_dp_chan >= 0) ? pack_mode4_or_5(mode45_block, pPixels, desired_dp_chan, mode6_sse_est, flags, &mode45_sse_est) : false; + BASISU_NOTE_UNUSED(mode45_valid_flag); + + uint8_t mode7_block[sizeof(basist::bc7_block)]; + bool mode7_valid_flag = false; + BASISU_NOTE_UNUSED(mode7_valid_flag); + + if ((flags & cPackBC7FlagUse2SubsetsRGBA) && (block_max_var4 >= MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH_RGBA); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + mode7_valid_flag = pack_mode7_rgba(mode7_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, mode6_sse_est, flags, &mode7_sse_est); + } + } + + if ((mode45_sse_est < mode7_sse_est) && (mode45_sse_est < mode6_sse_est)) + { + assert(mode45_valid_flag); + memcpy(pBlock, mode45_block, sizeof(basist::bc7_block)); + return; + } + else if ((mode7_sse_est < mode45_sse_est) && (mode7_sse_est < mode6_sse_est)) + { + assert(mode7_valid_flag); + memcpy(pBlock, mode7_block, sizeof(basist::bc7_block)); + return; + } + + // Fall back to mode 6 + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabs(w1)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + saxis_a = (int)((uint32_t)saxis_a << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + vec4F xl, xh; + bool res = compute_least_squares_endpoints_4D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b, (float)total_a); + + if (res) + { + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, xl[3] * q }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, xh[3] * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = (xl[3] >= 129.0f); + lr = to_7(xl[0], p0); + lg = to_7(xl[1], p0); + lb = to_7(xl[2], p0); + la = to_7(xl[3], p0); + + p1 = (xh[3] >= 129.0f); + hr = to_7(xh[0], p1); + hg = to_7(xh[1], p1); + hb = to_7(xh[2], p1); + ha = to_7(xh[3], p1); + } + + eval_weights_mode6_rgba(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + } + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + } + + uint32_t fast_pack_bc7_rgba_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + assert(g_bc7_4bit_ls_tab[1][0]); + +#if BASISU_BC7F_PERF_STATS + g_total_rgba_calls++; +#endif + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + pack_mode5_solid(pBlock, pPixels[0]); + return 0; + } + } + + int total_r = 0, total_g = 0, total_b = 0, total_a = 0; + int min_r = 255, min_g = 255, min_b = 255, min_a = 255; + int max_r = 0, max_g = 0, max_b = 0, max_a = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b, a = pPixels[i].a; + + total_r += r; total_g += g; total_b += b; total_a += a; + + min_r = basisu::minimum(min_r, r); min_g = basisu::minimum(min_g, g); min_b = basisu::minimum(min_b, b); min_a = basisu::minimum(min_a, a); + max_r = basisu::maximum(max_r, r); max_g = basisu::maximum(max_g, g); max_b = basisu::maximum(max_b, b); max_a = basisu::maximum(max_a, a); + } + + assert((min_r != max_r) || (min_g != max_g) || (min_b != max_b) || (min_a != max_a)); + + const int mean_r = (total_r + 8) >> 4, mean_g = (total_g + 8) >> 4, mean_b = (total_b + 8) >> 4, mean_a = (total_a + 8) >> 4; + + // covar rows are: + int icov4[10] = { }; + + // 0=rr + // 1=rg + // 2=rb + // 3=ra + // + // 4=gg + // 5=gb + // 6=ga + // + // 7=bb + // 8=ba + // + // 9=aa + + // 0 1 2 3 + // 4 5 6 + // 7 8 + // 9 + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // trace at 0,4,7,9 + + for (uint32_t i = 0; i < 16; i++) + { + const int r = (int)pPixels[i].r - mean_r, g = (int)pPixels[i].g - mean_g, b = (int)pPixels[i].b - mean_b, a = (int)pPixels[i].a - mean_a; + + icov4[0] += r * r; icov4[1] += r * g; icov4[2] += r * b; icov4[3] += r * a; + icov4[4] += g * g; icov4[5] += g * b; icov4[6] += g * a; + icov4[7] += b * b; icov4[8] += b * a; + icov4[9] += a * a; + } + + const int block_max_var4 = basisu::maximum(icov4[0], icov4[4], icov4[7], icov4[9]); // not divided by 16, i.e. scaled by 16 + assert(block_max_var4); // solid blocks already filtered out + + // check for dual plane, if a single component is very strongly decorrelated then switch to modes 4/5 + int desired_dp_chan = -1; + + const bool non_analytical_flag = (flags & cPackBC7FlagNonAnalyticalRGBA) != 0; + + if ((flags & cPackBC7FlagUseDualPlaneRGBA) && + ((!non_analytical_flag && (block_max_var4 >= DP_BLOCK_VAR_THRESH_RGBA)) || (non_analytical_flag && (block_max_var4 > 16))) + ) + { + // Prefer A, if not strongly decorrelated then check RGB. + const float r_var = (float)icov4[0], g_var = (float)icov4[4], b_var = (float)icov4[7], a_var = (float)icov4[9]; + + const bool has_a = icov4[9] > 0; + + if (has_a) + { + const float p_03 = icov4[0] ? fabs((float)icov4[3] / sqrtf(r_var * a_var)) : 1.0f; + const float p_13 = icov4[4] ? fabs((float)icov4[6] / sqrtf(g_var * a_var)) : 1.0f; + const float p_23 = icov4[7] ? fabs((float)icov4[8] / sqrtf(b_var * a_var)) : 1.0f; + + const float min_p = basisu::minimum(p_03, p_13, p_23); + if (min_p < ALPHA_DECORR_THRESHOLD) + { + desired_dp_chan = 3; +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_a++; +#endif + } + } + + if (desired_dp_chan < 0) + { + const bool has_r = icov4[0] > 16, has_g = icov4[4] > 16, has_b = icov4[7] > 16; + const uint32_t total_active_chans_rgb = has_r + has_g + has_b; + + if (total_active_chans_rgb >= 2) + { + const float rg_corr = (has_r && has_g) ? fabs((float)icov4[1] / sqrtf(r_var * g_var)) : 1.0f; + const float rb_corr = (has_r && has_b) ? fabs((float)icov4[2] / sqrtf(r_var * b_var)) : 1.0f; + const float gb_corr = (has_g && has_b) ? fabs((float)icov4[5] / sqrtf(g_var * b_var)) : 1.0f; + + float min_p = basisu::minimum(rg_corr, rb_corr, gb_corr); + + const float decorr_thresh = non_analytical_flag ? .999f : STRONG_DECORR_THRESH_RGBA; + if (min_p < decorr_thresh) + { + if (total_active_chans_rgb == 2) + { + if (!has_r) + desired_dp_chan = 1; + else if (!has_g) + desired_dp_chan = 0; + else + desired_dp_chan = 0; + } + else + { + // see if rg/rb is weakly correlated vs. gb + if ((rg_corr < gb_corr) && (rb_corr < gb_corr)) + desired_dp_chan = 0; + // see if gr/gb is weakly correlated vs. rb + else if ((rg_corr < rb_corr) && (gb_corr < rb_corr)) + desired_dp_chan = 1; + // assume b is weakest + else + desired_dp_chan = 2; + } +#if BASISU_BC7F_PERF_STATS + g_total_dp_valid_chans_rgb++; +#endif + } + } + } + } + + if ((flags & cPackBC7FlagUseTrivialMode6) && ((desired_dp_chan == -1) && (block_max_var4 < TRIVIAL_BLOCK_THRESH_RGBA))) + { + int low_c = INT_MAX, high_c = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int y = ((16 * 2) * pPixels[i].r + (16 * 4) * pPixels[i].g + 16 * pPixels[i].b + (16 * 4) * pPixels[i].a); + assert((y & 0xF) == 0); + y += i; + low_c = basisu::minimum(low_c, y); + high_c = basisu::maximum(high_c, y); + } + + low_c &= 0xF; + high_c &= 0xF; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + +#if BASISU_BC7F_PERF_STATS + g_total_trivial_mode6_blocks++; +#endif + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == mode6_actual_sse); + } +#endif + + return mode6_actual_sse; + } + + float cov4[10]; + for (uint32_t i = 0; i < 10; i++) + cov4[i] = (float)icov4[i]; + + // all channel pairs: + // 0,1=1 + // 0,2=2 + // 0,3=3 + // 1,2=5 + // 1,3=6 + // 2,3=8 + //const float r_var = cov4[0], g_var = cov4[4], b_var = cov4[7], a_var = cov4[9]; + + const float sc4 = block_max_var4 ? (1.0f / (float)block_max_var4) : 0; + float wx = sc4 * cov4[0], wy = sc4 * cov4[4], wz = sc4 * cov4[7], wa = sc4 * cov4[9]; + + // 0 1 2 3 + // 1 4 5 6 + // 2 5 7 8 + // 3 6 8 9 + + // TODO + float x1, y1, z1, w1; + for (uint32_t i = 0; i < 4; i++) + { + x1 = cov4[0] * wx + cov4[1] * wy + cov4[2] * wz + cov4[3] * wa; + y1 = cov4[1] * wx + cov4[4] * wy + cov4[5] * wz + cov4[6] * wa; + z1 = cov4[2] * wx + cov4[5] * wy + cov4[7] * wz + cov4[8] * wa; + w1 = cov4[3] * wx + cov4[6] * wy + cov4[8] * wz + cov4[9] * wa; + + float t = sqrtf(x1 * x1 + y1 * y1 + z1 * z1 + w1 * w1); + if (t > basisu::SMALL_FLOAT_VAL) + { + t = 1.0f / t; + x1 *= t; y1 *= t; z1 *= t; w1 *= t; + } + else + { + x1 = y1 = z1 = w1 = .25f; + } + + wx = x1; wy = y1; wz = z1; wa = w1; + } + + // Fall back to mode 6 + int saxis_r = 256, saxis_g = 256, saxis_b = 256, saxis_a = 256; + + float k = basisu::maximum(fabsf(x1), fabsf(y1), fabsf(z1), fabs(w1)); + if (fabs(k) >= basisu::SMALL_FLOAT_VAL) + { + float m = 2048.0f / k; + saxis_r = (int)(x1 * m); + saxis_g = (int)(y1 * m); + saxis_b = (int)(z1 * m); + saxis_a = (int)(w1 * m); + } + + saxis_r = (int)((uint32_t)saxis_r << 4U); + saxis_g = (int)((uint32_t)saxis_g << 4U); + saxis_b = (int)((uint32_t)saxis_b << 4U); + saxis_a = (int)((uint32_t)saxis_a << 4U); + + int low_dot = INT_MAX, high_dot = INT_MIN; + + for (uint32_t i = 0; i < 16; i += 4) + { + assert(((pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) & 0xF) == 0); // sanity + assert(((pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) & 0xF) == 0); + assert(((pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) & 0xF) == 0); + + const int dot0 = (pPixels[i].r * saxis_r + pPixels[i].g * saxis_g + pPixels[i].b * saxis_b + pPixels[i].a * saxis_a) + i; + const int dot1 = (pPixels[i + 1].r * saxis_r + pPixels[i + 1].g * saxis_g + pPixels[i + 1].b * saxis_b + pPixels[i + 1].a * saxis_a) + i + 1; + const int dot2 = (pPixels[i + 2].r * saxis_r + pPixels[i + 2].g * saxis_g + pPixels[i + 2].b * saxis_b + pPixels[i + 2].a * saxis_a) + i + 2; + const int dot3 = (pPixels[i + 3].r * saxis_r + pPixels[i + 3].g * saxis_g + pPixels[i + 3].b * saxis_b + pPixels[i + 3].a * saxis_a) + i + 3; + + int min_d01 = basisu::minimum(dot0, dot1); + int max_d01 = basisu::maximum(dot0, dot1); + + int min_d23 = basisu::minimum(dot2, dot3); + int max_d23 = basisu::maximum(dot2, dot3); + + int min_d = basisu::minimum(min_d01, min_d23); + int max_d = basisu::maximum(max_d01, max_d23); + + low_dot = basisu::minimum(low_dot, min_d); + high_dot = basisu::maximum(high_dot, max_d); + } + + const int low_c = low_dot & 15; + const int high_c = high_dot & 15; + + int p0, p1, lr, lg, lb, la, hr, hg, hb, ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { (float)pPixels[low_c].r * q, (float)pPixels[low_c].g * q, (float)pPixels[low_c].b * q, (float)pPixels[low_c].a * q }; + float sxh[4] = { (float)pPixels[high_c].r * q, (float)pPixels[high_c].g * q, (float)pPixels[high_c].b * q, (float)pPixels[high_c].a * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + p0 = best_pbits[0], p1 = best_pbits[1]; + lr = bestMinColor.r, lg = bestMinColor.g, lb = bestMinColor.b, la = bestMinColor.a; + hr = bestMaxColor.r, hg = bestMaxColor.g, hb = bestMaxColor.b, ha = bestMaxColor.a; + } + else + { + p0 = pPixels[low_c].a > 128; + p1 = pPixels[high_c].a > 128; + + lr = to_7(pPixels[low_c].r, p0), lg = to_7(pPixels[low_c].g, p0), lb = to_7(pPixels[low_c].b, p0), la = to_7(pPixels[low_c].a, p0); + hr = to_7(pPixels[high_c].r, p1), hg = to_7(pPixels[high_c].g, p1), hb = to_7(pPixels[high_c].b, p1), ha = to_7(pPixels[high_c].a, p1); + } + + uint8_t cur_weights[16]; + uint32_t mode6_actual_sse = eval_weights_mode6_rgba_sse(pPixels, cur_weights, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1); + + if (mode6_actual_sse) + { + vec4F xl, xh; + bool res = compute_least_squares_endpoints_4D( + 16, cur_weights, 16, + g_bc7_4bit_ls_tab, + xl, xh, + pPixels, + (float)total_r, (float)total_g, (float)total_b, (float)total_a); + + if (res) + { + int trial_p0, trial_p1, trial_lr, trial_lg, trial_lb, trial_la, trial_hr, trial_hg, trial_hb, trial_ha; + + if (flags & cPackBC7FlagPBitOptMode6) + { + const float q = 1.0f / 255.0f; + float sxl[4] = { xl[0] * q, xl[1] * q, xl[2] * q, xl[3] * q }; + float sxh[4] = { xh[0] * q, xh[1] * q, xh[2] * q, xh[3] * q }; + + color_rgba bestMinColor, bestMaxColor; + uint32_t best_pbits[2]; + determine_unique_pbits(4, 7, sxl, sxh, bestMinColor, bestMaxColor, best_pbits); + + trial_p0 = best_pbits[0], trial_p1 = best_pbits[1]; + trial_lr = bestMinColor.r, trial_lg = bestMinColor.g, trial_lb = bestMinColor.b, trial_la = bestMinColor.a; + trial_hr = bestMaxColor.r, trial_hg = bestMaxColor.g, trial_hb = bestMaxColor.b, trial_ha = bestMaxColor.a; + } + else + { + trial_p0 = (xl[3] >= 129.0f); + trial_lr = to_7(xl[0], trial_p0); + trial_lg = to_7(xl[1], trial_p0); + trial_lb = to_7(xl[2], trial_p0); + trial_la = to_7(xl[3], trial_p0); + + trial_p1 = (xh[3] >= 129.0f); + trial_hr = to_7(xh[0], trial_p1); + trial_hg = to_7(xh[1], trial_p1); + trial_hb = to_7(xh[2], trial_p1); + trial_ha = to_7(xh[3], trial_p1); + } + + uint8_t trial_weights[16]; + uint32_t mode6_ls_actual_sse = eval_weights_mode6_rgba_sse(pPixels, trial_weights, + trial_lr, trial_lg, trial_lb, trial_la, trial_p0, + trial_hr, trial_hg, trial_hb, trial_ha, trial_p1); + + if (mode6_ls_actual_sse < mode6_actual_sse) + { + mode6_actual_sse = mode6_ls_actual_sse; + memcpy(cur_weights, trial_weights, 16); + p0 = trial_p0; p1 = trial_p1; + lr = trial_lr; lg = trial_lg; lb = trial_lb; la = trial_la; + hr = trial_hr; hg = trial_hg; hb = trial_hb; ha = trial_ha; + } + } + } + + uint32_t mode7_actual_sse = UINT32_MAX; + uint8_t mode7_candidate_block[sizeof(basist::bc7_block)]; + + uint32_t mode45_actual_sse = UINT32_MAX; + uint8_t mode45_candidate_block[sizeof(basist::bc7_block)]; + + if (mode6_actual_sse) + { + if (non_analytical_flag) + { + // No gates: very expensive. + if (flags & cPackBC7FlagUse2SubsetsRGBA) + { + pack_mode7_rgba(mode7_candidate_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, 1e+9f, flags, nullptr, &mode7_actual_sse); + } + + if (flags & cPackBC7FlagUseDualPlaneRGBA) + pack_mode4_or_5(mode45_candidate_block, pPixels, (desired_dp_chan >= 0) ? desired_dp_chan : 3, 1e+9f, flags, nullptr, &mode45_actual_sse); // todo: determine best def channel here + } + else + { + float mode6_ortho_ratio; + const float mode6_slam_to_line_sse_est = estimate_slam_to_line_sse_4D(cov4, x1, y1, z1, w1, &mode6_ortho_ratio); + + if ((flags & cPackBC7FlagUse2SubsetsRGBA) && (block_max_var4 >= MIN_BLOCK_MAX_VAR_23SUBSETS_RGBA) && (mode6_ortho_ratio > ORTHO_RATIO_23SUBSET_RATIO_THRESH_RGBA)) + { + const bool high_ortho_energy_flag = (mode6_slam_to_line_sse_est >= HIGH_ORTHO_ENERGY_THRESH_RGBA); + + if (high_ortho_energy_flag) + { +#if BASISU_BC7F_PERF_STATS + g_total_high_ortho_energy++; +#endif + pack_mode7_rgba(mode7_candidate_block, pPixels, x1, y1, z1, w1, mean_r, mean_g, mean_b, mean_a, 1e+9f, flags, nullptr, &mode7_actual_sse); + } + } + + if (desired_dp_chan >= 0) + pack_mode4_or_5(mode45_candidate_block, pPixels, desired_dp_chan, 1e+9f, flags, nullptr, &mode45_actual_sse); + } + } + + const uint32_t best_actual_sse = basisu::minimum(mode6_actual_sse, mode45_actual_sse, mode7_actual_sse); + + if ((mode45_actual_sse != UINT32_MAX) && (best_actual_sse == mode45_actual_sse)) + { + memcpy(pBlock, mode45_candidate_block, sizeof(basist::bc7_block)); + } + else if ((mode7_actual_sse != UINT32_MAX) && (best_actual_sse == mode7_actual_sse)) + { + memcpy(pBlock, mode7_candidate_block, sizeof(basist::bc7_block)); + } + else + { + assert(mode6_actual_sse == best_actual_sse); + + encode_mode6_rgba_block(pBlock, + lr, lg, lb, la, p0, + hr, hg, hb, ha, p1, + cur_weights); + } + +#ifdef _DEBUG + { + // Final sanity checking. + uint32_t expected_actual_sse = calc_sse(pBlock, pPixels); + assert(expected_actual_sse == best_actual_sse); + } +#endif + + return best_actual_sse; + } + + // Routes to either rgb or rgba automatically + uint32_t fast_pack_bc7_auto_rgba(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { + for (uint32_t i = 0; i < 16; i += 4) + { + if ((pPixels[i].a < 255) || (pPixels[i + 1].a < 255) || (pPixels[i + 2].a < 255) || (pPixels[i + 3].a < 255)) + { + if (flags & cPackBC7FlagPartiallyAnalyticalRGBA) + return bc7f::fast_pack_bc7_rgba_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgba_analytical(pBlock, pPixels, flags); + return 0; + } + } + } + + if (flags & cPackBC7FlagPartiallyAnalyticalRGB) + return bc7f::fast_pack_bc7_rgb_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgb_analytical(pBlock, pPixels, flags); + return 0; + } + } + + // Source block cannot have alpha. + uint32_t fast_pack_bc7_auto_rgb(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags) + { +// Disabling this check here, because during fuzzing the ktx2 file may lie. In this case it's harmless to output an opaque block. +#if 0 +#if defined(DEBUG) || defined(_DEBUG) + for (uint32_t i = 0; i < 16; i += 4) + { + if ((pPixels[i].a < 255) || (pPixels[i + 1].a < 255) || (pPixels[i + 2].a < 255) || (pPixels[i + 3].a < 255)) + { + // Block can't have alpha here, or the solid color detectors may misfire. + assert(0); + } + } +#endif +#endif + + if (flags & cPackBC7FlagPartiallyAnalyticalRGB) + return bc7f::fast_pack_bc7_rgb_partial_analytical(pBlock, pPixels, flags); + else + { + bc7f::fast_pack_bc7_rgb_analytical(pBlock, pPixels, flags); + return 0; + } + } + + void clear_perf_stats() + { +#if BASISU_BC7F_PERF_STATS +#define BU_CLEAR_BLOCK_STAT(x) x = 0; + BU_CLEAR_BLOCK_STAT(g_total_rgb_calls); + BU_CLEAR_BLOCK_STAT(g_total_rgba_calls); + BU_CLEAR_BLOCK_STAT(g_total_solid_blocks); + BU_CLEAR_BLOCK_STAT(g_total_trivial_mode6_blocks); + BU_CLEAR_BLOCK_STAT(g_total_dp_valid_chans_rgb); + BU_CLEAR_BLOCK_STAT(g_total_dp_valid_chans_a); + BU_CLEAR_BLOCK_STAT(g_total_high_ortho_energy); + BU_CLEAR_BLOCK_STAT(g_total_mode02_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode02_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode13_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode13_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode45_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode45_bailouts); + BU_CLEAR_BLOCK_STAT(g_total_mode7_evals); + BU_CLEAR_BLOCK_STAT(g_total_mode7_bailouts); +#undef BU_CLEAR_BLOCK_STAT +#endif + } + + void print_perf_stats() + { +#if BASISU_BC7F_PERF_STATS + const uint32_t total_bc7_blocks = g_total_rgb_calls + g_total_rgba_calls; + if (!total_bc7_blocks) + return; + +#define BU_PRINT_BLOCK_STAT(x) printf(#x ": %u %3.2f%%\n", (uint32_t)x, static_cast(x) * 100.0f / (float)total_bc7_blocks); + BU_PRINT_BLOCK_STAT(g_total_rgb_calls); + BU_PRINT_BLOCK_STAT(g_total_rgba_calls); + BU_PRINT_BLOCK_STAT(g_total_solid_blocks); + BU_PRINT_BLOCK_STAT(g_total_trivial_mode6_blocks); + BU_PRINT_BLOCK_STAT(g_total_dp_valid_chans_rgb); + BU_PRINT_BLOCK_STAT(g_total_dp_valid_chans_a); + BU_PRINT_BLOCK_STAT(g_total_high_ortho_energy); + BU_PRINT_BLOCK_STAT(g_total_mode02_evals); + BU_PRINT_BLOCK_STAT(g_total_mode02_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode13_evals); + BU_PRINT_BLOCK_STAT(g_total_mode13_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode45_evals); + BU_PRINT_BLOCK_STAT(g_total_mode45_bailouts); + BU_PRINT_BLOCK_STAT(g_total_mode7_evals); + BU_PRINT_BLOCK_STAT(g_total_mode7_bailouts); +#undef BU_PRINT_BLOCK_STAT + +#endif + } + +#if 0 + struct bc7_mode_6 + { + struct + { + uint64_t m_mode : 7; + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 7; + uint64_t m_a1 : 7; + uint64_t m_p0 : 1; + } m_lo; + + union + { + struct + { + uint64_t m_p1 : 1; + uint64_t m_s00 : 3; + uint64_t m_s10 : 4; + uint64_t m_s20 : 4; + uint64_t m_s30 : 4; + + uint64_t m_s01 : 4; + uint64_t m_s11 : 4; + uint64_t m_s21 : 4; + uint64_t m_s31 : 4; + + uint64_t m_s02 : 4; + uint64_t m_s12 : 4; + uint64_t m_s22 : 4; + uint64_t m_s32 : 4; + + uint64_t m_s03 : 4; + uint64_t m_s13 : 4; + uint64_t m_s23 : 4; + uint64_t m_s33 : 4; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; +#endif + +#if 0 + // Very basic ASTC LDR 4x4 packer which transcodes BC7 mode 6 RGB/RGBA only to ASTC LDR 4x4. + void fast_pack_astc(void* pBlock, const color_rgba* pPixels) + { + astc_helpers::astc_block* pDst_block = (astc_helpers::astc_block*)pBlock; + + astc_helpers::log_astc_block log_blk; + log_blk.clear(); + log_blk.m_grid_width = 4; + log_blk.m_grid_height = 4; + + const uint32_t fc = *(const uint32_t*)&pPixels[0]; + if (fc == *(const uint32_t*)&pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (*(const uint32_t*)&pPixels[k] != fc) + break; + + if (k == 15) + { + const uint32_t r = pPixels[0].r, g = pPixels[0].g, b = pPixels[0].b, a = pPixels[0].a; + + log_blk.m_solid_color_flag_ldr = true; + log_blk.m_solid_color[0] = (uint16_t)(r | ((uint32_t)r << 8)); + log_blk.m_solid_color[1] = (uint16_t)(g | ((uint32_t)g << 8)); + log_blk.m_solid_color[2] = (uint16_t)(b | ((uint32_t)b << 8)); + log_blk.m_solid_color[3] = (uint16_t)(a | ((uint32_t)a << 8)); + + bool pack_status = astc_helpers::pack_astc_block(*pDst_block, log_blk); + assert(pack_status); + BASISU_NOTE_UNUSED(pack_status); + + return; + } + } + + basist::bc7_block bc7_block; + fast_pack_bc7_auto_rgba((uint8_t*)&bc7_block, pPixels, cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6 | cPackBC7FlagUseTrivialMode6); + + assert(bc7u::determine_bc7_mode(&bc7_block) == 6); + + bc7u::bc7_mode_6& mode6 = *(bc7u::bc7_mode_6*)&bc7_block; + + log_blk.m_num_partitions = 1; + + if ((mode6.m_lo.m_a0 < 127) || (mode6.m_lo.m_a1 < 127)) + { + log_blk.m_color_endpoint_modes[0] = 12; + log_blk.m_endpoint_ise_range = astc_helpers::BISE_96_LEVELS; + log_blk.m_weight_ise_range = astc_helpers::BISE_12_LEVELS; + + const auto& endpoint_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_val_to_ise; + const auto& endpoint_from_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_val; + //const auto& weight_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + int p0 = mode6.m_lo.m_p0; + int r0 = bc7f::from_7(mode6.m_lo.m_r0, p0); + int g0 = bc7f::from_7(mode6.m_lo.m_g0, p0); + int b0 = bc7f::from_7(mode6.m_lo.m_b0, p0); + int a0 = bc7f::from_7(mode6.m_lo.m_a0, p0); + + int p1 = mode6.m_hi.m_p1; + int r1 = bc7f::from_7(mode6.m_lo.m_r1, p1); + int g1 = bc7f::from_7(mode6.m_lo.m_g1, p1); + int b1 = bc7f::from_7(mode6.m_lo.m_b1, p1); + int a1 = bc7f::from_7(mode6.m_lo.m_a1, p1); + + log_blk.m_endpoints[0] = endpoint_to_ise[r0]; + log_blk.m_endpoints[1] = endpoint_to_ise[r1]; + + log_blk.m_endpoints[2] = endpoint_to_ise[g0]; + log_blk.m_endpoints[3] = endpoint_to_ise[g1]; + + log_blk.m_endpoints[4] = endpoint_to_ise[b0]; + log_blk.m_endpoints[5] = endpoint_to_ise[b1]; + + log_blk.m_endpoints[6] = endpoint_to_ise[a0]; + log_blk.m_endpoints[7] = endpoint_to_ise[a1]; + + int s0 = endpoint_from_ise[log_blk.m_endpoints[0]] + endpoint_from_ise[log_blk.m_endpoints[2]] + endpoint_from_ise[log_blk.m_endpoints[4]]; + int s1 = endpoint_from_ise[log_blk.m_endpoints[1]] + endpoint_from_ise[log_blk.m_endpoints[3]] + endpoint_from_ise[log_blk.m_endpoints[5]]; + + int invw = 0; + if (s1 < s0) + { + std::swap(log_blk.m_endpoints[0], log_blk.m_endpoints[1]); + std::swap(log_blk.m_endpoints[2], log_blk.m_endpoints[3]); + std::swap(log_blk.m_endpoints[4], log_blk.m_endpoints[5]); + std::swap(log_blk.m_endpoints[6], log_blk.m_endpoints[7]); + std::swap(g0, g1); + std::swap(b0, b1); + invw = 15; + } + + static const uint8_t s_pWeight_to_ise[16] = { 0, 4, 8, 8, 2, 6, 10, 10, 11, 11, 7, 3, 9, 9, 5, 1 }; + + log_blk.m_weights[0] = s_pWeight_to_ise[mode6.m_hi.m_s00 ^ invw]; + log_blk.m_weights[1] = s_pWeight_to_ise[mode6.m_hi.m_s10 ^ invw]; + log_blk.m_weights[2] = s_pWeight_to_ise[mode6.m_hi.m_s20 ^ invw]; + log_blk.m_weights[3] = s_pWeight_to_ise[mode6.m_hi.m_s30 ^ invw]; + + log_blk.m_weights[4] = s_pWeight_to_ise[mode6.m_hi.m_s01 ^ invw]; + log_blk.m_weights[5] = s_pWeight_to_ise[mode6.m_hi.m_s11 ^ invw]; + log_blk.m_weights[6] = s_pWeight_to_ise[mode6.m_hi.m_s21 ^ invw]; + log_blk.m_weights[7] = s_pWeight_to_ise[mode6.m_hi.m_s31 ^ invw]; + + log_blk.m_weights[8] = s_pWeight_to_ise[mode6.m_hi.m_s02 ^ invw]; + log_blk.m_weights[9] = s_pWeight_to_ise[mode6.m_hi.m_s12 ^ invw]; + log_blk.m_weights[10] = s_pWeight_to_ise[mode6.m_hi.m_s22 ^ invw]; + log_blk.m_weights[11] = s_pWeight_to_ise[mode6.m_hi.m_s32 ^ invw]; + + log_blk.m_weights[12] = s_pWeight_to_ise[mode6.m_hi.m_s03 ^ invw]; + log_blk.m_weights[13] = s_pWeight_to_ise[mode6.m_hi.m_s13 ^ invw]; + log_blk.m_weights[14] = s_pWeight_to_ise[mode6.m_hi.m_s23 ^ invw]; + log_blk.m_weights[15] = s_pWeight_to_ise[mode6.m_hi.m_s33 ^ invw]; + } + else + { + log_blk.m_color_endpoint_modes[0] = 8; + log_blk.m_endpoint_ise_range = astc_helpers::BISE_192_LEVELS; + log_blk.m_weight_ise_range = astc_helpers::BISE_16_LEVELS; + + const auto& endpoint_to_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_val_to_ise; + const auto& endpoint_from_ise = astc_helpers::g_dequant_tables.get_endpoint_tab(log_blk.m_endpoint_ise_range).m_ISE_to_val; + const auto& weight_to_ise = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range).m_rank_to_ISE; + + int p0 = mode6.m_lo.m_p0; + int r0 = bc7f::from_7(mode6.m_lo.m_r0, p0); + int g0 = bc7f::from_7(mode6.m_lo.m_g0, p0); + int b0 = bc7f::from_7(mode6.m_lo.m_b0, p0); + + int p1 = mode6.m_hi.m_p1; + int r1 = bc7f::from_7(mode6.m_lo.m_r1, p1); + int g1 = bc7f::from_7(mode6.m_lo.m_g1, p1); + int b1 = bc7f::from_7(mode6.m_lo.m_b1, p1); + + log_blk.m_endpoints[0] = endpoint_to_ise[r0]; + log_blk.m_endpoints[1] = endpoint_to_ise[r1]; + + log_blk.m_endpoints[2] = endpoint_to_ise[g0]; + log_blk.m_endpoints[3] = endpoint_to_ise[g1]; + + log_blk.m_endpoints[4] = endpoint_to_ise[b0]; + log_blk.m_endpoints[5] = endpoint_to_ise[b1]; + + int s0 = endpoint_from_ise[log_blk.m_endpoints[0]] + endpoint_from_ise[log_blk.m_endpoints[2]] + endpoint_from_ise[log_blk.m_endpoints[4]]; + int s1 = endpoint_from_ise[log_blk.m_endpoints[1]] + endpoint_from_ise[log_blk.m_endpoints[3]] + endpoint_from_ise[log_blk.m_endpoints[5]]; + + int invw = 0; + if (s1 < s0) + { + std::swap(log_blk.m_endpoints[0], log_blk.m_endpoints[1]); + std::swap(log_blk.m_endpoints[2], log_blk.m_endpoints[3]); + std::swap(log_blk.m_endpoints[4], log_blk.m_endpoints[5]); + invw = 15; + } + + log_blk.m_weights[0] = weight_to_ise[(size_t)(mode6.m_hi.m_s00 ^ invw)]; + log_blk.m_weights[1] = weight_to_ise[(size_t)(mode6.m_hi.m_s10 ^ invw)]; + log_blk.m_weights[2] = weight_to_ise[(size_t)(mode6.m_hi.m_s20 ^ invw)]; + log_blk.m_weights[3] = weight_to_ise[(size_t)(mode6.m_hi.m_s30 ^ invw)]; + + log_blk.m_weights[4] = weight_to_ise[(size_t)(mode6.m_hi.m_s01 ^ invw)]; + log_blk.m_weights[5] = weight_to_ise[(size_t)(mode6.m_hi.m_s11 ^ invw)]; + log_blk.m_weights[6] = weight_to_ise[(size_t)(mode6.m_hi.m_s21 ^ invw)]; + log_blk.m_weights[7] = weight_to_ise[(size_t)(mode6.m_hi.m_s31 ^ invw)]; + + log_blk.m_weights[8] = weight_to_ise[(size_t)(mode6.m_hi.m_s02 ^ invw)]; + log_blk.m_weights[9] = weight_to_ise[(size_t)(mode6.m_hi.m_s12 ^ invw)]; + log_blk.m_weights[10] = weight_to_ise[(size_t)(mode6.m_hi.m_s22 ^ invw)]; + log_blk.m_weights[11] = weight_to_ise[(size_t)(mode6.m_hi.m_s32 ^ invw)]; + + log_blk.m_weights[12] = weight_to_ise[(size_t)(mode6.m_hi.m_s03 ^ invw)]; + log_blk.m_weights[13] = weight_to_ise[(size_t)(mode6.m_hi.m_s13 ^ invw)]; + log_blk.m_weights[14] = weight_to_ise[(size_t)(mode6.m_hi.m_s23 ^ invw)]; + log_blk.m_weights[15] = weight_to_ise[(size_t)(mode6.m_hi.m_s33 ^ invw)]; + } + + bool pack_status = astc_helpers::pack_astc_block(*pDst_block, log_blk); + assert(pack_status); + BASISU_NOTE_UNUSED(pack_status); + } +#endif + +} // namespace bc7f + +namespace etc1f +{ +#include "basisu_etc1_mods.inl" + + // flip 0: + // 0011 + // 0011 + // 0011 + // 0011 + + // flip 1: + // 0000 + // 0000 + // 1111 + // 1111 + + uint8_t g_nearest5[256], g_nearest4[256]; + + const uint32_t NUM_SOLID_MODS = 4; + + uint8_t g_solid8_5_base[256][NUM_SOLID_MODS][4]; // [desired8][mod][sel] + uint8_t g_solid8_5_err[256][NUM_SOLID_MODS][4]; + uint8_t g_solid8_4_base[256][NUM_SOLID_MODS][4]; + uint8_t g_solid8_4_err[256][NUM_SOLID_MODS][4]; + + uint8_t g_solid_grayscale_etc1_blocks[256][8]; + + inline int expand5(int v5) + { + return (v5 << 3) | (v5 >> 2); + } + + inline int expand4(int v4) + { + return (v4 << 4) | v4; + } + + static inline int dequant4(uint32_t v) + { + assert(v < 16); + return (v << 4) | v; + } + + static inline int dequant5(uint32_t v) + { + assert(v < 32); + return (v << 3) | (v >> 2); + } + + void init() + { + for (int i = 0; i < 256; i++) + { + int best_e = INT_MAX, best_idx = 0; + + for (int s = 0; s < 32; s++) + { + int recovered = (s << 3) | (s >> 2); + int e = basisu::iabs(recovered - i); + if (e < best_e) + { + best_e = e; + best_idx = s; + } + } + + g_nearest5[i] = (uint8_t)best_idx; + } + + for (int i = 0; i < 256; i++) + { + int best_e = INT_MAX, best_idx = 0; + + for (int s = 0; s < 16; s++) + { + int recovered = (s << 4) | s; + int e = basisu::iabs(recovered - i); + if (e < best_e) + { + best_e = e; + best_idx = s; + } + } + + g_nearest4[i] = (uint8_t)best_idx; + } + + for (uint32_t desired8 = 0; desired8 < 256; desired8++) + { + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + int32_t best_err = INT32_MAX; + uint32_t best_base = 0; + + for (uint32_t b = 0; b < 32; b++) + { + int val = basisu::clamp(dequant5(b) + g_etc1_inten_tables[mod][sel], 0, 255); + int err = basisu::iabs(val - desired8); + + if (err < best_err) + { + best_err = err; + best_base = b; + if (!best_err) + break; + } + + } // b + + g_solid8_5_base[desired8][mod][sel] = (uint8_t)best_base; + g_solid8_5_err[desired8][mod][sel] = (uint8_t)basisu::minimum(255, best_err * best_err); + + } // sel + + } // mod + + } // desired8 + + for (uint32_t desired8 = 0; desired8 < 256; desired8++) + { + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + int32_t best_err = INT32_MAX; + uint32_t best_base = 0; + + for (uint32_t b = 0; b < 16; b++) + { + int val = basisu::clamp(dequant4(b) + g_etc1_inten_tables[mod][sel], 0, 255); + int err = basisu::iabs(val - desired8); + + if (err < best_err) + { + best_err = err; + best_base = b; + if (!best_err) + break; + } + + } // b + + g_solid8_4_base[desired8][mod][sel] = (uint8_t)best_base; + g_solid8_4_err[desired8][mod][sel] = (uint8_t)basisu::minimum(255, best_err * best_err); + + } // sel + + } // mod + + } // desired8 + + pack_etc1_state pack_state; + + for (uint32_t i = 0; i <= 255; i++) + { + etc1f::pack_etc1_solid(&g_solid_grayscale_etc1_blocks[i][0], color_rgba(i, i, i, 255), pack_state, true); + } + } + + inline int dequant_d3(int8_t v) + { + assert(v <= 7); + return (int8_t(v << 5) >> 5); + } + + void get_block_colors(uint8_t* pBlock, color_rgba* pColors0, color_rgba* pColors1) + { + const uint32_t b0 = pBlock[0], b1 = pBlock[1], b2 = pBlock[2], b3 = pBlock[3]; + + int base8_r[2], base8_g[2], base8_b[2]; + + if (b3 & 2) + { + // diff mode + base8_r[0] = dequant5(b0 >> 3); + base8_r[1] = dequant5(basisu::clamp((b0 >> 3) + dequant_d3(b0 & 7), 0, 31)); + + base8_g[0] = dequant5(b1 >> 3); + base8_g[1] = dequant5(basisu::clamp((b1 >> 3) + dequant_d3(b1 & 7), 0, 31)); + + base8_b[0] = dequant5(b2 >> 3); + base8_b[1] = dequant5(basisu::clamp((b2 >> 3) + dequant_d3(b2 & 7), 0, 31)); + } + else + { + // abs mode + base8_r[0] = dequant4(b0 >> 4); + base8_r[1] = dequant4(b0 & 15); + + base8_g[0] = dequant4(b1 >> 4); + base8_g[1] = dequant4(b1 & 15); + + base8_b[0] = dequant4(b2 >> 4); + base8_b[1] = dequant4(b2 & 15); + } + + const int* pInten_table0 = &g_etc1_inten_tables[b3 >> 5][0]; + const int* pInten_table1 = &g_etc1_inten_tables[(b3 >> 2) & 7][0]; + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table0[i]; + pColors0[i].r = (uint8_t)clamp255(base8_r[0] + d); + pColors0[i].g = (uint8_t)clamp255(base8_g[0] + d); + pColors0[i].b = (uint8_t)clamp255(base8_b[0] + d); + pColors0[i].a = 0; + } + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table1[i]; + pColors1[i].r = (uint8_t)clamp255(base8_r[1] + d); + pColors1[i].g = (uint8_t)clamp255(base8_g[1] + d); + pColors1[i].b = (uint8_t)clamp255(base8_b[1] + d); + pColors1[i].a = 0; + } + } + + void get_block_colors_y(uint8_t* pBlock, uint8_t* pColors0, uint8_t* pColors1) + { + //const uint32_t b0 = pBlock[0], b1 = pBlock[1], b2 = pBlock[2], b3 = pBlock[3]; + const uint32_t b0 = pBlock[0], b3 = pBlock[3]; + + int base8_y[2]; + + if (b3 & 2) + { + // diff mode + base8_y[0] = dequant5(b0 >> 3); + base8_y[1] = dequant5(basisu::clamp((b0 >> 3) + dequant_d3(b0 & 7), 0, 31)); + } + else + { + // abs mode + base8_y[0] = dequant4(b0 >> 4); + base8_y[1] = dequant4(b0 & 15); + } + + const int* pInten_table0 = g_etc1_inten_tables[b3 >> 5]; + const int* pInten_table1 = g_etc1_inten_tables[(b3 >> 2) & 7]; + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table0[i]; + pColors0[i] = (uint8_t)clamp255(base8_y[0] + d); + } + + for (uint32_t i = 0; i < 4; i++) + { + const int d = pInten_table1[i]; + pColors1[i] = (uint8_t)clamp255(base8_y[1] + d); + } + } + + static inline int q4_floor(int x) { return (x * 15) / 255; } + static inline int q5_floor(int x) { return (x * 31) / 255; } + + void corr_round_555(int R, int G, int B, int& qR, int& qG, int& qB) + { + int rL = q5_floor(R), gL = q5_floor(G), bL = q5_floor(B); + int rH = (rL < 31) ? (rL + 1) : 31; + int gH = (gL < 31) ? (gL + 1) : 31; + int bH = (bL < 31) ? (bL + 1) : 31; + + int r8[2] = { expand5(rL), expand5(rH) }; + int g8[2] = { expand5(gL), expand5(gH) }; + int b8[2] = { expand5(bL), expand5(bH) }; + + int tr = r8[0], tg = g8[0], tb = b8[0]; + int eR = R - tr, eG = G - tg, eB = B - tb; + int bestJ = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + int br = tr, bg = tg, bb = tb; + + for (int m = 1; m < 8; ++m) + { + tr = r8[m & 1], tg = g8[(m >> 1) & 1], tb = b8[(m >> 2) & 1]; + eR = R - tr, eG = G - tg, eB = B - tb; + + int J = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + if (J < bestJ) + { + bestJ = J; + br = tr; + bg = tg; + bb = tb; + } + } + + qR = br >> 3; qG = bg >> 3; qB = bb >> 3; + } + + void corr_round_444(int R, int G, int B, int& qR, int& qG, int& qB) + { + int rL = q4_floor(R), gL = q4_floor(G), bL = q4_floor(B); + int rH = (rL < 15) ? (rL + 1) : 15; + int gH = (gL < 15) ? (gL + 1) : 15; + int bH = (bL < 15) ? (bL + 1) : 15; + + int r8[2] = { expand4(rL), expand4(rH) }; + int g8[2] = { expand4(gL), expand4(gH) }; + int b8[2] = { expand4(bL), expand4(bH) }; + + int tr = r8[0], tg = g8[0], tb = b8[0]; + int eR = R - tr, eG = G - tg, eB = B - tb; + int bestJ = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + int br = tr, bg = tg, bb = tb; + + for (int m = 1; m < 8; ++m) + { + tr = r8[m & 1], tg = g8[(m >> 1) & 1], tb = b8[(m >> 2) & 1]; + eR = R - tr, eG = G - tg, eB = B - tb; + + int J = basisu::squarei(eR - eG) + basisu::squarei(eG - eB) + basisu::squarei(eB - eR); + if (J < bestJ) + { + bestJ = J; + br = tr; + bg = tg; + bb = tb; + } + } + + qR = br >> 4; qG = bg >> 4; qB = bb >> 4; + } + + inline bool quantize_444_color_correlated(int mean8_r, int mean8_g, int mean8_b, int enc_color[3], bool early_out = true) + { + // Floor to low 4-bit + int r4_low = (mean8_r * 15) / 255; + int g4_low = (mean8_g * 15) / 255; + int b4_low = (mean8_b * 15) / 255; + + // High = +1, clamped + int r4_high = basisu::clamp(r4_low + 1, 0, 15); + int g4_high = basisu::clamp(g4_low + 1, 0, 15); + int b4_high = basisu::clamp(b4_low + 1, 0, 15); + + const int r8_low = expand4(r4_low); + const int g8_low = expand4(g4_low); + const int b8_low = expand4(b4_low); + const int r8_high = expand4(r4_high); + const int g8_high = expand4(g4_high); + const int b8_high = expand4(b4_high); + + // Errors if we pick "low" + const float dr = float(r8_low) - mean8_r; + const float dg = float(g8_low) - mean8_g; + const float db = float(b8_low) - mean8_b; + + const float dRG = fabsf(dr - dg); + const float dRB = fabsf(dr - db); + const float dGB = fabsf(dg - db); + const float maxSpread = basisu::maximum(dRG, dRB, dGB); + + if ((early_out) && (maxSpread <= 1.0f)) + return false; + + // Step sizes low->high + const float kr = float(r8_high - r8_low); + const float kg = float(g8_high - g8_low); + const float kb = float(b8_high - b8_low); + + // Precompute constants for cost(mask) = S0 + Lsum - Ksum^2 + const float D = dr + dg + db; + const float S0 = 3.0f * (dr * dr + dg * dg + db * db) - D * D; + + const float Lr = 6.0f * dr * kr + 3.0f * kr * kr - 2.0f * D * kr; + const float Lg = 6.0f * dg * kg + 3.0f * kg * kg - 2.0f * D * kg; + const float Lb = 6.0f * db * kb + 3.0f * kb * kb - 2.0f * D * kb; + + float bestCost = basisu::BIG_FLOAT_VAL; + int bestMask = 0; + + for (int mask = 0; mask < 8; ++mask) + { + const float Ksum = ((mask & 1) ? kr : 0.0f) + ((mask & 2) ? kg : 0.0f) + ((mask & 4) ? kb : 0.0f); + const float Lsum = ((mask & 1) ? Lr : 0.0f) + ((mask & 2) ? Lg : 0.0f) + ((mask & 4) ? Lb : 0.0f); + + const float cost = S0 + Lsum - Ksum * Ksum; + + if (cost < bestCost) + { + bestCost = cost; + bestMask = mask; + } + } + + enc_color[0] = (bestMask & 1) ? r4_high : r4_low; + enc_color[1] = (bestMask & 2) ? g4_high : g4_low; + enc_color[2] = (bestMask & 4) ? b4_high : b4_low; + + assert((enc_color[0]) >= 0 && (enc_color[0] <= 15)); + assert((enc_color[1]) >= 0 && (enc_color[1] <= 15)); + assert((enc_color[2]) >= 0 && (enc_color[2] <= 15)); + + return true; + } + + void pack_etc1_solid(uint8_t* pBlock, const color_rgba& color, pack_etc1_state& state, bool init_flag) + { + uint32_t r8 = color[0], g8 = color[1], b8 = color[2]; + //const uint32_t r8 = 0, g8 = 0, b8 = 0; + + if (!init_flag) + { + if ((r8 == g8) && (r8 == b8)) + { + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[r8][0], sizeof(decoder_etc_block)); + return; + } + + if ((state.m_prev_solid_r8 == (int)r8) && (state.m_prev_solid_g8 == (int)g8) && (state.m_prev_solid_b8 == (int)b8)) + { + memcpy(pBlock, &state.m_prev_solid_block, sizeof(decoder_etc_block)); + return; + } + } + + uint32_t best_err = UINT32_MAX; + uint32_t best_mod = 0, best_sel = 0; + uint32_t best4_flag = false; + + const int RW = 2, GW = 4; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + for (uint32_t sel = 0; sel < 4; sel++) + { + uint32_t total_err5 = RW * g_solid8_5_err[r8][mod][sel] + GW * g_solid8_5_err[g8][mod][sel] + g_solid8_5_err[b8][mod][sel]; + if (total_err5 < best_err) + { + best_err = total_err5; + best_mod = mod; + best_sel = sel; + best4_flag = false; + if (!best_err) + goto etc1_solid_done; + } + + uint32_t total_err4 = RW * g_solid8_4_err[r8][mod][sel] + GW * g_solid8_4_err[g8][mod][sel] + g_solid8_4_err[b8][mod][sel]; + if (total_err4 < best_err) + { + best_err = total_err4; + best_mod = mod; + best_sel = sel; + best4_flag = true; + } + + } // sel + } // mod + + etc1_solid_done: + + if (best4_flag) + { + const uint32_t best_base_r4 = g_solid8_4_base[r8][best_mod][best_sel]; + const uint32_t best_base_g4 = g_solid8_4_base[g8][best_mod][best_sel]; + const uint32_t best_base_b4 = g_solid8_4_base[b8][best_mod][best_sel]; + + pBlock[0] = (uint8_t)(best_base_r4 | (best_base_r4 << 4)); + pBlock[1] = (uint8_t)(best_base_g4 | (best_base_g4 << 4)); + pBlock[2] = (uint8_t)(best_base_b4 | (best_base_b4 << 4)); + } + else + { + pBlock[0] = (uint8_t)(g_solid8_5_base[r8][best_mod][best_sel] << 3); + pBlock[1] = (uint8_t)(g_solid8_5_base[g8][best_mod][best_sel] << 3); + pBlock[2] = (uint8_t)(g_solid8_5_base[b8][best_mod][best_sel] << 3); + } + + const uint32_t flip = 0; + const uint32_t diff = (best4_flag == 0); + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod << 5) | (best_mod << 2)); + + const uint32_t etc1_sels = g_selector_index_to_etc1[best_sel]; + + const uint8_t lb = (etc1_sels & 2) ? 0xFF : 0; + pBlock[4] = lb; + pBlock[5] = lb; + + const uint8_t hb = (etc1_sels & 1) ? 0xFF : 0; + pBlock[6] = hb; + pBlock[7] = hb; + + state.m_prev_solid_r8 = r8; + state.m_prev_solid_g8 = g8; + state.m_prev_solid_b8 = b8; + memcpy(&state.m_prev_solid_block, pBlock, sizeof(decoder_etc_block)); + } + + static const uint8_t s_vi[16] = { 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }; + static const uint8_t s_hi[16] = { 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 }; + + static const uint8_t s_subsets[2][16] = + { + { 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1 }, + { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 } + }; + + // [flip][subblock][sel][l/h] + static const uint16_t s_sel_bitmasks[2 * 2 * 4][2] = + { + // flip=0, subblock=0, sels=0-3 + { 0xff, 0xff }, + { 0x0, 0xff }, + { 0x0, 0x0 }, + { 0xff, 0x0 }, + + // flip=0, subblock=1, sels=0-3 + { 0xff00, 0xff00 }, + { 0x0, 0xff00 }, + { 0x0, 0x0 }, + { 0xff00, 0x0 }, + + // flip=1, subblock=0, sels=0-3 + { 0x3333, 0x3333 }, + { 0x0, 0x3333 }, + { 0x0, 0x0 }, + { 0x3333, 0x0 }, + + // flip=1, subblock=1, sels=0-3 + { 0xcccc, 0xcccc }, + { 0x0, 0xcccc }, + { 0x0, 0x0 }, + { 0xcccc, 0x0 } + }; + + void pack_etc1_solid_subblocks(uint8_t* pBlock, const color_rgba* pPixels, const color_rgba subblock_means[2], uint32_t flip) + { + (void)pPixels; + + uint32_t best_mod5[2] = {}, best_sel5[2] = {}; + uint32_t best_base5[2][3] = {}; + uint32_t best_err5[2] = { UINT32_MAX, UINT32_MAX }; + + uint32_t best_mod4[2] = {}, best_sel4[2] = {}; + uint32_t best_base4[2][3] = {}; + uint32_t best_err4[2] = { UINT32_MAX, UINT32_MAX }; + + const int RW = 2, GW = 4; + + for (uint32_t t = 0; t < 2; t++) + { + const uint32_t r8 = subblock_means[t][0]; + const uint32_t g8 = subblock_means[t][1]; + const uint32_t b8 = subblock_means[t][2]; + + const uint8_t* pR5 = &g_solid8_5_err[r8][0][0]; + const uint8_t* pG5 = &g_solid8_5_err[g8][0][0]; + const uint8_t* pB5 = &g_solid8_5_err[b8][0][0]; + + const uint8_t* pR4 = &g_solid8_4_err[r8][0][0]; + const uint8_t* pG4 = &g_solid8_4_err[g8][0][0]; + const uint8_t* pB4 = &g_solid8_4_err[b8][0][0]; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + const uint32_t mod4 = mod << 2; + + const uint32_t total_err5_0 = ((RW * pR5[0] + GW * pG5[0] + pB5[0]) << 5) + (mod4 + 0); + const uint32_t total_err5_1 = ((RW * pR5[1] + GW * pG5[1] + pB5[1]) << 5) + (mod4 + 1); + const uint32_t total_err5_2 = ((RW * pR5[2] + GW * pG5[2] + pB5[2]) << 5) + (mod4 + 2); + const uint32_t total_err5_3 = ((RW * pR5[3] + GW * pG5[3] + pB5[3]) << 5) + (mod4 + 3); + + best_err5[t] = basisu::minimum(best_err5[t], basisu::minimum(total_err5_0, total_err5_1), basisu::minimum(total_err5_2, total_err5_3)); + + const uint32_t total_err4_0 = ((RW * pR4[0] + GW * pG4[0] + pB4[0]) << 5) + (mod4 + 0); + const uint32_t total_err4_1 = ((RW * pR4[1] + GW * pG4[1] + pB4[1]) << 5) + (mod4 + 1); + const uint32_t total_err4_2 = ((RW * pR4[2] + GW * pG4[2] + pB4[2]) << 5) + (mod4 + 2); + const uint32_t total_err4_3 = ((RW * pR4[3] + GW * pG4[3] + pB4[3]) << 5) + (mod4 + 3); + + best_err4[t] = basisu::minimum(best_err4[t], basisu::minimum(total_err4_0, total_err4_1), basisu::minimum(total_err4_2, total_err4_3)); + + pR5 += 4; pG5 += 4; pB5 += 4; + pR4 += 4; pG4 += 4; pB4 += 4; + } // mod + + best_mod5[t] = (best_err5[t] >> 2) & 7; + best_sel5[t] = best_err5[t] & 3; + best_err5[t] >>= 5; + + best_mod4[t] = (best_err4[t] >> 2) & 7; + best_sel4[t] = best_err4[t] & 3; + best_err4[t] >>= 5; + + best_base5[t][0] = g_solid8_5_base[r8][best_mod5[t]][best_sel5[t]]; + best_base5[t][1] = g_solid8_5_base[g8][best_mod5[t]][best_sel5[t]]; + best_base5[t][2] = g_solid8_5_base[b8][best_mod5[t]][best_sel5[t]]; + + best_base4[t][0] = g_solid8_4_base[r8][best_mod4[t]][best_sel4[t]]; + best_base4[t][1] = g_solid8_4_base[g8][best_mod4[t]][best_sel4[t]]; + best_base4[t][2] = g_solid8_4_base[b8][best_mod4[t]][best_sel4[t]]; + + } // t + + uint32_t total_err4 = best_err4[0] + best_err4[1]; + uint32_t total_err5 = best_err5[0] + best_err5[1]; + + bool use_abs = false; + if (total_err4 < total_err5) + { + use_abs = true; + } + else + { + int delta_r = best_base5[1][0] - best_base5[0][0]; + int delta_g = best_base5[1][1] - best_base5[0][1]; + int delta_b = best_base5[1][2] - best_base5[0][2]; + + if ((delta_r < -4) || (delta_r > 3) || + (delta_g < -4) || (delta_g > 3) || + (delta_b < -4) || (delta_b > 3)) + { + use_abs = true; + } + } + + uint32_t* pBest_sels; + + if (use_abs) + { + pBlock[0] = (uint8_t)(best_base4[1][0] | (best_base4[0][0] << 4)); + pBlock[1] = (uint8_t)(best_base4[1][1] | (best_base4[0][1] << 4)); + pBlock[2] = (uint8_t)(best_base4[1][2] | (best_base4[0][2] << 4)); + + const uint32_t diff = false; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod4[0] << 5) | (best_mod4[1] << 2)); + + pBest_sels = best_sel4; + } + else + { + const int delta_r = (best_base5[1][0] - best_base5[0][0]) & 7; + const int delta_g = (best_base5[1][1] - best_base5[0][1]) & 7; + const int delta_b = (best_base5[1][2] - best_base5[0][2]) & 7; + + pBlock[0] = (uint8_t)(delta_r | (best_base5[0][0] << 3)); + pBlock[1] = (uint8_t)(delta_g | (best_base5[0][1] << 3)); + pBlock[2] = (uint8_t)(delta_b | (best_base5[0][2] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod5[0] << 5) | (best_mod5[1] << 2)); + + pBest_sels = best_sel5; + } + + uint16_t l_bitmask = 0, h_bitmask = 0; + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + uint32_t best_etc1_sel = pBest_sels[subblock]; + + l_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][0]; + h_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][1]; + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + } + + //------------------------------------------ + + void pack_etc1(uint8_t* pBlock, const color_rgba* pPixels, pack_etc1_state& state) + { + { + // Solid block check, ignoring alpha. + const uint32_t fc = *(const uint32_t*)&pPixels[0] & BASISD_COLOR_RGBA_RGB_MASK; + + if (fc == (*(const uint32_t*)&pPixels[15] & BASISD_COLOR_RGBA_RGB_MASK)) + { + int k; + for (k = 1; k < 15; k++) + if ((*(const uint32_t*)&pPixels[k] & BASISD_COLOR_RGBA_RGB_MASK) != fc) + break; + + if (k == 15) + { + pack_etc1_solid(pBlock, pPixels[0], state, false); + return; + } + } + } + + // [0]=left, [1]=right, [2]=top, [3]=bottom + int accum_y[4] = { 0 }, accum_y2[4] = { 0 }, accum_c2[4] = { 0 }; + int total_c2 = 0, max_c2 = 0; + + for (uint32_t i = 0; i < 16; i++) + { + int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + int rg = r - g, bg = b - g; + + int y = (r + g + b + 1) / 3; + int y2 = y * y, c2 = rg * rg + bg * bg; + + total_c2 += c2; + max_c2 = basisu::maximum(max_c2, c2); + + const int vi = s_vi[i], hi = s_hi[i]; + + accum_y[vi] += y; + accum_y2[vi] += y2; + accum_c2[vi] += c2; + + accum_y[hi] += y; + accum_y2[hi] += y2; + accum_c2[hi] += c2; + + } // i + +#if 1 + // sqrt(300/16)=~4.33 + const int CHROMA_ENERGY_SUM_THRESH = 300; + const int CHROMA_ENERGY_MAX_THRESH = 32; + if ((total_c2 < CHROMA_ENERGY_SUM_THRESH) && (max_c2 < CHROMA_ENERGY_MAX_THRESH)) + { + //memset(pBlock, 0, 8); + //return; + + uint8_t y_pixels[16]; + if (total_c2 == 0) + { + for (uint32_t i = 0; i < 16; i++) + y_pixels[i] = pPixels[i].r; + } + else + { + for (uint32_t i = 0; i < 16; i++) + y_pixels[i] = (uint8_t)pPixels[i].get_709_luma(); + } + pack_etc1_grayscale(pBlock, y_pixels, state); + return; + } +#endif + + int var_y_scaled[4]; // scaled by x64 (8*8) + for (uint32_t i = 0; i < 4; i++) + var_y_scaled[i] = basisu::maximum(0, (accum_y2[i] << 3) - (accum_y[i] * accum_y[i])); // max not needed + + float std_luma[4], std_chroma[4]; + for (uint32_t i = 0; i < 4; i++) + { + std_luma[i] = sqrtf((float)var_y_scaled[i] * (1.0f / 64.0f)); + std_chroma[i] = sqrtf((float)accum_c2[i] * (1.0f / 8.0f)); + } + + const float LUMA_SCALE = 2, CHROMA_SCALE = 1; + float flip0_score = (std_luma[0] + std_luma[1]) * LUMA_SCALE + (std_chroma[0] + std_chroma[1]) * CHROMA_SCALE; + float flip1_score = (std_luma[2] + std_luma[3]) * LUMA_SCALE + (std_chroma[2] + std_chroma[3]) * CHROMA_SCALE; + + const uint32_t flip = flip1_score < flip0_score; + + int var8_y[2] = {}, mean8_y[2] = {}, mean8_r[2] = {}, mean8_g[2] = {}, mean8_b[2] = {}; + int min_y[2] = { INT_MAX, INT_MAX }, max_y[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int r = pPixels[i].r, g = pPixels[i].g, b = pPixels[i].b; + const int y = (r + g + b + 1) / 3; + + const uint32_t s = s_subsets[flip][i]; + + var8_y[s] += y * y; + mean8_y[s] += y; + + mean8_r[s] += r; + mean8_g[s] += g; + mean8_b[s] += b; + + min_y[s] = basisu::minimum(min_y[s], y); + max_y[s] = basisu::maximum(max_y[s], y); + } + + //memset(pBlock, 0, sizeof(etc_block)); + //return; + + if (((max_y[0] - min_y[0]) < 8) && ((max_y[1] - min_y[1]) < 8)) + { + color_rgba subblock_means[2] = { + color_rgba((mean8_r[0] + 4) / 8, (mean8_g[0] + 4) / 8, (mean8_b[0] + 4) / 8, 255), + color_rgba((mean8_r[1] + 4) / 8, (mean8_g[1] + 4) / 8, (mean8_b[1] + 4) / 8, 255), + }; + + if (subblock_means[0] == subblock_means[1]) + pack_etc1_solid(pBlock, subblock_means[0], state, false); + else + pack_etc1_solid_subblocks(pBlock, pPixels, subblock_means, flip); + + return; + } + + //memset(pBlock, 0, sizeof(etc_block)); + //return; + + int half_span8_y[2]; + float stddev_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + var8_y[i] = basisu::maximum(0, (var8_y[i] << 3) - mean8_y[i] * mean8_y[i]); + stddev_y[i] = std::sqrt(static_cast(var8_y[i])) * (1.0f / 8.0f); + + mean8_y[i] = (mean8_y[i] + 4) >> 3; + + mean8_r[i] = (mean8_r[i] + 4) >> 3; + mean8_g[i] = (mean8_g[i] + 4) >> 3; + mean8_b[i] = (mean8_b[i] + 4) >> 3; + + half_span8_y[i] = basisu::maximum(max_y[i] - mean8_y[i], mean8_y[i] - min_y[i]); + } + + int stddev[2] = + { + basisu::clamp((int)ceilf(9.0f * (stddev_y[0] / (float)basisu::maximum(1, half_span8_y[0]))) - 1, 0, 7), + basisu::clamp((int)ceilf(9.0f * (stddev_y[1] / (float)basisu::maximum(1, half_span8_y[1]))) - 1, 0, 7) + }; + + uint32_t mod_tab[2]; + for (uint32_t i = 0; i < 2; i++) + mod_tab[i] = g_etc1_mod_tabs[basisu::clamp(half_span8_y[i], 1, 255)][stddev[i]]; + + int mean5_r[2], mean5_g[2], mean5_b[2]; + + for (uint32_t i = 0; i < 2; i++) + { +#if 1 + corr_round_555(mean8_r[i], mean8_g[i], mean8_b[i], mean5_r[i], mean5_g[i], mean5_b[i]); +#else + mean5_r[i] = g_nearest5[mean8_r[i]]; + mean5_g[i] = g_nearest5[mean8_g[i]]; + mean5_b[i] = g_nearest5[mean8_b[i]]; +#endif + } + + int delta5_r = mean5_r[1] - mean5_r[0]; + int delta5_g = mean5_g[1] - mean5_g[0]; + int delta5_b = mean5_b[1] - mean5_b[0]; + + const uint32_t z = (delta5_r + 4) | (delta5_g + 4) | (delta5_b + 4); + bool use_abs_colors4 = z > 7; + + if (!use_abs_colors4) + { + assert((delta5_r >= -4) && (delta5_r <= 3)); + assert((delta5_g >= -4) && (delta5_g <= 3)); + assert((delta5_b >= -4) && (delta5_b <= 3)); + } + + if (use_abs_colors4) + { + int mean4_r[2], mean4_g[2], mean4_b[2]; + + for (uint32_t i = 0; i < 2; i++) + { +#if 1 + corr_round_444(mean8_r[i], mean8_g[i], mean8_b[i], mean4_r[i], mean4_g[i], mean4_b[i]); +#else + mean4_r[i] = g_nearest4[mean8_r[i]]; + mean4_g[i] = g_nearest4[mean8_g[i]]; + mean4_b[i] = g_nearest4[mean8_b[i]]; +#endif + } // i + + pBlock[0] = (uint8_t)(mean4_r[1] | (mean4_r[0] << 4)); + pBlock[1] = (uint8_t)(mean4_g[1] | (mean4_g[0] << 4)); + pBlock[2] = (uint8_t)(mean4_b[1] | (mean4_b[0] << 4)); + + const uint32_t diff = 0; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + else + { + pBlock[0] = (uint8_t)((delta5_r & 7) | (mean5_r[0] << 3)); + pBlock[1] = (uint8_t)((delta5_g & 7) | (mean5_g[0] << 3)); + pBlock[2] = (uint8_t)((delta5_b & 7) | (mean5_b[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + + uint16_t l_bitmask = 0; + uint16_t h_bitmask = 0; + + static const uint8_t s_tran[4] = { 1, 0, 2, 3 }; + + color_rgba subblock_colors[2][4]; + get_block_colors(pBlock, &subblock_colors[0][0], &subblock_colors[1][0]); + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + const color_rgba* block_colors = &subblock_colors[subblock][0]; + + uint32_t block_y[4]; + for (uint32_t i = 0; i < 4; i++) + block_y[i] = block_colors[i][0] * 54 + block_colors[i][1] * 183 + block_colors[i][2] * 19; + + const uint32_t block_y01 = block_y[0] + block_y[1]; + const uint32_t block_y12 = block_y[1] + block_y[2]; + const uint32_t block_y23 = block_y[2] + block_y[3]; + + if (flip) + { + uint32_t ofs = subblock * 2; + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + const color_rgba& c = pPixels[x + (subblock * 2 + y) * 4]; + const uint32_t l = c[0] * 108 + c[1] * 366 + c[2] * 38; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ofs += 4; + } + + ofs = (int)ofs + 1 - 4 * 4; + } + } + else + { + uint32_t ofs = (subblock * 2) * 4; + for (uint32_t x = 0; x < 2; x++) + { + for (uint32_t y = 0; y < 4; y++) + { + const color_rgba& c = pPixels[subblock * 2 + x + y * 4]; + const uint32_t l = c[0] * 108 + c[1] * 366 + c[2] * 38; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ++ofs; + } + } + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + + } // subblock + } + + void pack_etc1_grayscale_solid_subblocks(uint8_t* pBlock, const uint8_t* pPixels, const uint8_t subblock_means[2], uint32_t flip) + { + (void)pPixels; + + uint32_t best_mod5[2] = {}, best_sel5[2] = {}; + uint32_t best_base5[2] = {}; + uint32_t best_err5[2] = { UINT32_MAX, UINT32_MAX }; + + uint32_t best_mod4[2] = {}, best_sel4[2] = {}; + uint32_t best_base4[2] = {}; + uint32_t best_err4[2] = { UINT32_MAX, UINT32_MAX }; + + for (uint32_t t = 0; t < 2; t++) + { + const uint32_t y8 = subblock_means[t]; + + const uint8_t* pY5 = &g_solid8_5_err[y8][0][0]; + const uint8_t* pY4 = &g_solid8_4_err[y8][0][0]; + + for (uint32_t mod = 0; mod < NUM_SOLID_MODS; mod++) + { + const uint32_t mod4 = mod << 2; + + const uint32_t total_err5_0 = (pY5[0] << 5) + (mod4 + 0); + const uint32_t total_err5_1 = (pY5[1] << 5) + (mod4 + 1); + const uint32_t total_err5_2 = (pY5[2] << 5) + (mod4 + 2); + const uint32_t total_err5_3 = (pY5[3] << 5) + (mod4 + 3); + + best_err5[t] = basisu::minimum(best_err5[t], basisu::minimum(total_err5_0, total_err5_1), basisu::minimum(total_err5_2, total_err5_3)); + + const uint32_t total_err4_0 = (pY4[0] << 5) + (mod4 + 0); + const uint32_t total_err4_1 = (pY4[1] << 5) + (mod4 + 1); + const uint32_t total_err4_2 = (pY4[2] << 5) + (mod4 + 2); + const uint32_t total_err4_3 = (pY4[3] << 5) + (mod4 + 3); + + best_err4[t] = basisu::minimum(best_err4[t], basisu::minimum(total_err4_0, total_err4_1), basisu::minimum(total_err4_2, total_err4_3)); + + pY5 += 4; + pY4 += 4; + } // mod + + best_mod5[t] = (best_err5[t] >> 2) & 7; + best_sel5[t] = best_err5[t] & 3; + best_err5[t] >>= 5; + + best_mod4[t] = (best_err4[t] >> 2) & 7; + best_sel4[t] = best_err4[t] & 3; + best_err4[t] >>= 5; + + best_base5[t] = g_solid8_5_base[y8][best_mod5[t]][best_sel5[t]]; + + best_base4[t] = g_solid8_4_base[y8][best_mod4[t]][best_sel4[t]]; + + } // t + + uint32_t total_err4 = best_err4[0] + best_err4[1]; + uint32_t total_err5 = best_err5[0] + best_err5[1]; + + bool use_abs = false; + if (total_err4 < total_err5) + { + use_abs = true; + } + else + { + int delta_y = best_base5[1] - best_base5[0]; + + if ((delta_y < -4) || (delta_y > 3)) + { + use_abs = true; + } + } + + uint32_t* pBest_sels; + + if (use_abs) + { + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(best_base4[1] | (best_base4[0] << 4)); + + const uint32_t diff = false; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod4[0] << 5) | (best_mod4[1] << 2)); + + pBest_sels = best_sel4; + } + else + { + const int delta_y = (best_base5[1] - best_base5[0]) & 7; + + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(delta_y | (best_base5[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (best_mod5[0] << 5) | (best_mod5[1] << 2)); + + pBest_sels = best_sel5; + } + + uint16_t l_bitmask = 0, h_bitmask = 0; + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + uint32_t best_etc1_sel = pBest_sels[subblock]; + + l_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][0]; + h_bitmask |= s_sel_bitmasks[flip * 8 + subblock * 4 + best_etc1_sel][1]; + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + } + + void pack_etc1_grayscale(uint8_t* pBlock, const uint8_t* pPixels, pack_etc1_state& state) + { + (void)state; + + const uint8_t fc = pPixels[0]; + + if (fc == pPixels[15]) + { + int k; + for (k = 1; k < 15; k++) + if (pPixels[k] != fc) + break; + + if (k == 15) + { + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[fc][0], sizeof(decoder_etc_block)); + return; + } + } + + int accum_y[4] = { 0 }, accum_y2[4] = { 0 }; + + for (uint32_t i = 0; i < 16; i++) + { + int y = pPixels[i]; + int y2 = y * y; + + const int vi = s_vi[i], hi = s_hi[i]; + + accum_y[vi] += y; + accum_y2[vi] += y2; + + accum_y[hi] += y; + accum_y2[hi] += y2; + + } // i + + int var_y_scaled[4]; // scaled by x64 (8*8) + for (uint32_t i = 0; i < 4; i++) + var_y_scaled[i] = basisu::maximum(0, (accum_y2[i] << 3) - (accum_y[i] * accum_y[i])); // max not needed + + float std_luma[4]; + for (uint32_t i = 0; i < 4; i++) + std_luma[i] = sqrtf((float)var_y_scaled[i] * (1.0f / 64.0f)); + + float flip0_score = std_luma[0] + std_luma[1]; + float flip1_score = std_luma[2] + std_luma[3]; + + const uint32_t flip = flip1_score < flip0_score; + + int var8_y[2] = {}, mean8_y[2] = {}; + int min_y[2] = { INT_MAX, INT_MAX }, max_y[2] = { INT_MIN, INT_MIN }; + + for (uint32_t i = 0; i < 16; i++) + { + const int y = pPixels[i]; + + const uint32_t s = s_subsets[flip][i]; + + var8_y[s] += y * y; + mean8_y[s] += y; + + min_y[s] = basisu::minimum(min_y[s], y); + max_y[s] = basisu::maximum(max_y[s], y); + } + + if (((max_y[0] - min_y[0]) < 8) && ((max_y[1] - min_y[1]) < 8)) + { + uint8_t subblock_means[2] = { + (uint8_t)((mean8_y[0] + 4) / 8), + (uint8_t)((mean8_y[1] + 4) / 8), + }; + + if (subblock_means[0] == subblock_means[1]) + memcpy(pBlock, &g_solid_grayscale_etc1_blocks[subblock_means[0]][0], sizeof(decoder_etc_block)); + else + pack_etc1_grayscale_solid_subblocks(pBlock, pPixels, subblock_means, flip); + + return; + } + + int half_span8_y[2]; + float stddev_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + var8_y[i] = basisu::maximum(0, (var8_y[i] << 3) - mean8_y[i] * mean8_y[i]); + stddev_y[i] = std::sqrt(static_cast(var8_y[i])) * (1.0f / 8.0f); + + mean8_y[i] = (mean8_y[i] + 4) >> 3; + + half_span8_y[i] = basisu::maximum(max_y[i] - mean8_y[i], mean8_y[i] - min_y[i]); + } + + int stddev[2] = + { + basisu::clamp((int)ceilf(9.0f * (stddev_y[0] / (float)basisu::maximum(1, half_span8_y[0]))) - 1, 0, 7), + basisu::clamp((int)ceilf(9.0f * (stddev_y[1] / (float)basisu::maximum(1, half_span8_y[1]))) - 1, 0, 7) + }; + + uint32_t mod_tab[2]; + for (uint32_t i = 0; i < 2; i++) + mod_tab[i] = etc1f::g_etc1_mod_tabs[basisu::clamp(half_span8_y[i], 1, 255)][stddev[i]]; + + int mean5_y[2]; + + for (uint32_t i = 0; i < 2; i++) + mean5_y[i] = g_nearest5[mean8_y[i]]; + + int delta5_y = mean5_y[1] - mean5_y[0]; + + const uint32_t z = delta5_y + 4; + bool use_abs_colors4 = z > 7; + + if (!use_abs_colors4) + { + assert((delta5_y >= -4) && (delta5_y <= 3)); + } + + if (use_abs_colors4) + { + int mean4_y[2]; + + for (uint32_t i = 0; i < 2; i++) + { + mean4_y[i] = g_nearest4[mean8_y[i]]; + } // i + + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)(mean4_y[1] | (mean4_y[0] << 4)); + + const uint32_t diff = 0; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + else + { + pBlock[0] = pBlock[1] = pBlock[2] = (uint8_t)((delta5_y & 7) | (mean5_y[0] << 3)); + + const uint32_t diff = 1; + pBlock[3] = (uint8_t)(flip | (diff << 1) | (mod_tab[0] << 5) | (mod_tab[1] << 2)); + } + + //decoder_etc_block& blk = *(decoder_etc_block*)pBlock; + + uint16_t l_bitmask = 0; + uint16_t h_bitmask = 0; + + static const uint8_t s_tran[4] = { 1, 0, 2, 3 }; + + uint8_t subblock_colors[2][4]; + get_block_colors_y(pBlock, &subblock_colors[0][0], &subblock_colors[1][0]); + + for (uint32_t subblock = 0; subblock < 2; subblock++) + { + const uint8_t* block_y = &subblock_colors[subblock][0]; + + //color_rgba block_colors[4]; + //blk.get_block_colors(block_colors, subblock); + + //uint32_t block_y[4]; + //for (uint32_t i = 0; i < 4; i++) + //block_y[i] = block_colors[i]; + + const uint32_t block_y01 = block_y[0] + block_y[1]; + const uint32_t block_y12 = block_y[1] + block_y[2]; + const uint32_t block_y23 = block_y[2] + block_y[3]; + + if (flip) + { + uint32_t ofs = subblock * 2; + + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 4; x++) + { + const uint8_t c = pPixels[x + (subblock * 2 + y) * 4]; + const uint32_t l = c * 2; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ofs += 4; + } + + ofs = (int)ofs + 1 - 4 * 4; + } + } + else + { + uint32_t ofs = (subblock * 2) * 4; + for (uint32_t x = 0; x < 2; x++) + { + for (uint32_t y = 0; y < 4; y++) + { + const uint8_t c = pPixels[subblock * 2 + x + y * 4]; + const uint32_t l = c * 2; + + uint32_t t = s_tran[(l < block_y01) + (l < block_y12) + (l < block_y23)]; + + assert(ofs < 16); + l_bitmask |= ((t & 1) << ofs); + h_bitmask |= ((t >> 1) << ofs); + ++ofs; + } + } + } + + pBlock[7] = (uint8_t)(l_bitmask); + pBlock[6] = (uint8_t)(l_bitmask >> 8); + pBlock[5] = (uint8_t)(h_bitmask); + pBlock[4] = (uint8_t)(h_bitmask >> 8); + + } // subblock + } + +} // namespace etc1f + +#endif // BASISD_SUPPORT_XUASTC + +//------------------------------------------------------------------------------------------------ +// XUASTC LDR transcoding +//------------------------------------------------------------------------------------------------ +// XUASTC adaptive deblocking threshold +const int XUASTC_LDR_DEBLOCK_SKIP_THRESH = 24; + +block_format xuastc_get_block_format(transcoder_texture_format tex_fmt) +{ + switch (tex_fmt) + { + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: return block_format::cASTC_LDR_4x4; + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: return block_format::cASTC_LDR_5x4; + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: return block_format::cASTC_LDR_5x5; + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: return block_format::cASTC_LDR_6x5; + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: return block_format::cASTC_LDR_6x6; + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: return block_format::cASTC_LDR_8x5; + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: return block_format::cASTC_LDR_8x6; + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: return block_format::cASTC_LDR_10x5; + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: return block_format::cASTC_LDR_10x6; + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: return block_format::cASTC_LDR_8x8; + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: return block_format::cASTC_LDR_10x8; + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: return block_format::cASTC_LDR_10x10; + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: return block_format::cASTC_LDR_12x10; + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: return block_format::cASTC_LDR_12x12; + default: + break; + } + + assert(0); + return block_format::cASTC_LDR_4x4; +} + +basisu_lowlevel_xuastc_ldr_transcoder::basisu_lowlevel_xuastc_ldr_transcoder() +{ +} + +#if BASISD_SUPPORT_XUASTC +void transcode_4x4_block( + block_format fmt, + uint32_t block_x, uint32_t block_y, + void *pDst_blocks, uint8_t* pDst_block_u8, + const color32* block_pixels, + uint32_t output_block_or_pixel_stride_in_bytes, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, + int channel0, int channel1, + bool high_quality, bool from_alpha, + uint32_t bc7f_flags, + etc1f::pack_etc1_state& etc1_pack_state, + int has_alpha) // has_alpha = -1 unknown, 0=definitely no (a all 255's), 1=potentially yes +{ + BASISU_NOTE_UNUSED(output_block_or_pixel_stride_in_bytes); + + switch (fmt) + { + case block_format::cETC1: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + if (from_alpha) + { + // Annoying overhead + uint8_t alpha_pixels[16]; + for (uint32_t i = 0; i < 16; i++) + alpha_pixels[i] = block_pixels[i].a; + + etc1f::pack_etc1_grayscale(pDst_block_u8, alpha_pixels, etc1_pack_state); + } + else + { + etc1f::pack_etc1(pDst_block_u8, (color_rgba *)block_pixels, etc1_pack_state); + } + break; + } + case block_format::cETC2_RGBA: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[3], sizeof(color32)); + etc1f::pack_etc1(pDst_block_u8 + 8, (color_rgba*)block_pixels, etc1_pack_state); + + break; + } + case block_format::cETC2_EAC_R11: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + // Pack R by default + if (channel0 < 0) + channel0 = 0; + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[channel0], sizeof(color32)); + + break; + } + case block_format::cETC2_EAC_RG11: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // Pack RA by default + if (channel0 < 0) + channel0 = 0; + if (channel1 < 0) + channel1 = 3; + + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[0], &block_pixels[0].c[channel0], sizeof(color32)); + (high_quality ? pack_eac_high_quality : pack_eac)(reinterpret_cast(pDst_block_u8)[1], &block_pixels[0].c[channel1], sizeof(color32)); + + break; + } + case block_format::cBC1: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + encode_bc1(pDst_block_u8, (const uint8_t *)block_pixels, high_quality ? cEncodeBC1HighQuality : 0); + break; + } + case block_format::cBC3: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + encode_bc4(pDst_block_u8, &block_pixels[0].c[3], sizeof(color32)); + encode_bc1(pDst_block_u8 + 8, (const uint8_t *)block_pixels, high_quality ? cEncodeBC1HighQuality : 0); + break; + } + case block_format::cBC4: + { + assert(output_block_or_pixel_stride_in_bytes == 8); + + // Pack R by default + if (channel0 < 0) + channel0 = 0; + + encode_bc4(pDst_block_u8, &block_pixels[0].c[channel0], sizeof(color32)); + break; + } + case block_format::cBC5: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // Pack RA by default + if (channel0 < 0) + channel0 = 0; + if (channel1 < 0) + channel1 = 3; + + encode_bc4(pDst_block_u8, &block_pixels[0].c[channel0], sizeof(color32)); + encode_bc4(pDst_block_u8 + 8, &block_pixels[0].c[channel1], sizeof(color32)); + + break; + } + case block_format::cBC7: + { + assert(output_block_or_pixel_stride_in_bytes == 16); + + // 0=definitely no alpha, so skip alpha checks + if (has_alpha == 0) + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, (const basist::color_rgba*)block_pixels, bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, (const basist::color_rgba*)block_pixels, bc7f_flags); + + break; + } + case block_format::cRGBA32: + { + assert(sizeof(uint32_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint32_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + if ((max_x == 4) && (max_y == 4)) + { + memcpy(pDst_pixels, block_pixels, 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 1, &block_pixels[1 * 4], 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 2, &block_pixels[2 * 4], 4 * sizeof(color32)); + memcpy(pDst_pixels + output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t) * 3, &block_pixels[3 * 4], 4 * sizeof(color32)); + } + else + { + for (uint32_t y = 0; y < max_y; y++) + { + memcpy(pDst_pixels, &block_pixels[y * 4], max_x * sizeof(color32)); + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint32_t); + } + } + + break; + } + case block_format::cRGB565: + case block_format::cBGR565: + { + // This writes little endian data always. + assert(sizeof(uint16_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint16_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + for (uint32_t y = 0; y < max_y; y++) + { + for (uint32_t x = 0; x < max_x; x++) + { + const color32& c = block_pixels[y * 4 + x]; + + const uint16_t packed = (fmt == block_format::cRGB565) ? static_cast((mul_8(c.r, 31) << 11) | (mul_8(c.g, 63) << 5) | mul_8(c.b, 31)) : + static_cast((mul_8(c.b, 31) << 11) | (mul_8(c.g, 63) << 5) | mul_8(c.r, 31)); + + pDst_pixels[x * 2 + 0] = (uint8_t)(packed & 0xFF); + pDst_pixels[x * 2 + 1] = (uint8_t)((packed >> 8) & 0xFF); + } + + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint16_t); + } + + break; + } + case block_format::cRGBA4444: + { + // This writes little endian data always. + assert(sizeof(uint16_t) == output_block_or_pixel_stride_in_bytes); + uint8_t* pDst_pixels = static_cast(pDst_blocks) + (block_x * 4 + block_y * 4 * output_row_pitch_in_blocks_or_pixels) * sizeof(uint16_t); + + const uint32_t max_x = basisu::minimum(4, (int)output_row_pitch_in_blocks_or_pixels - (int)block_x * 4); + const uint32_t max_y = basisu::minimum(4, (int)output_rows_in_pixels - (int)block_y * 4); + + for (uint32_t y = 0; y < max_y; y++) + { + for (uint32_t x = 0; x < max_x; x++) + { + const color32& c = block_pixels[y * 4 + x]; + + const uint16_t packed = static_cast((mul_8(c.r, 15) << 12) | (mul_8(c.g, 15) << 8) | (mul_8(c.b, 15) << 4) | mul_8(c.a, 15)); + + pDst_pixels[x * 2 + 0] = (uint8_t)(packed & 0xFF); + pDst_pixels[x * 2 + 1] = (uint8_t)((packed >> 8) & 0xFF); + } + + pDst_pixels += output_row_pitch_in_blocks_or_pixels * sizeof(uint16_t); + } + break; + } + default: + // Unsupported or invalid format + assert(0); + break; + } +} + +static bool xuastc_deblock_filter( + uint32_t filter_block_width, uint32_t filter_block_height, + const basisu::vector2D &src_img, + basisu::vector2D &dst_img, + bool stronger_filtering, int skip_thresh) +{ + basisu::vector2D temp_img; + if (!temp_img.try_resize(src_img.get_width(), src_img.get_height())) + return false; + + if (stronger_filtering) + skip_thresh *= 2; + + //basisu::fmt_printf("stronger filtering: {}, skip_thread: {}\n", stronger_filtering, skip_thresh); + + temp_img = src_img; + + for (int y = 0; y < (int)src_img.get_height(); y++) + { + for (int x = filter_block_width; x < (int)src_img.get_width(); x += filter_block_width) + { + const color32 &ll = src_img.at_clamped(x - 2, y); + const color32 &l = src_img.at_clamped(x - 1, y); + const color32 &r =src_img.at_clamped(x, y); + const color32 &rr = src_img.at_clamped(x + 1, y); + + if (skip_thresh < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = basisu::iabs((int)l[c] - (int)r[c]); + if (delta > skip_thresh) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color32 ml, mr; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + ml[c] = (3 * l[c] + 2 * r[c] + ll[c] + 3) / 6; + mr[c] = (3 * r[c] + 2 * l[c] + rr[c] + 3) / 6; + } + else + { + ml[c] = (5 * l[c] + 2 * r[c] + ll[c] + 4) / 8; + mr[c] = (5 * r[c] + 2 * l[c] + rr[c] + 4) / 8; + } + } + + temp_img.set_clipped(x - 1, y, ml); + temp_img.set_clipped(x, y, mr); + + } // x + + } // y + + dst_img = temp_img; + + for (int x = 0; x < (int)temp_img.get_width(); x++) + { + for (int y = filter_block_height; y < (int)temp_img.get_height(); y += filter_block_height) + { + const color32 &uu = temp_img.at_clamped(x, y - 2); + const color32 &u = temp_img.at_clamped(x, y - 1); + const color32 &d = temp_img.at_clamped(x, y); + const color32 &dd = temp_img.at_clamped(x, y + 1); + + if (skip_thresh < 256) + { + bool skip_flag = false; + for (uint32_t c = 0; c < 4; c++) + { + int delta = basisu::iabs((int)u[c] - (int)d[c]); + if (delta > skip_thresh) + { + skip_flag = true; + break; + } + } + + if (skip_flag) + continue; + } + + color32 mu, md; + for (uint32_t c = 0; c < 4; c++) + { + if (stronger_filtering) + { + mu[c] = (3 * u[c] + 2 * d[c] + uu[c] + 3) / 6; + md[c] = (3 * d[c] + 2 * u[c] + dd[c] + 3) / 6; + } + else + { + mu[c] = (5 * u[c] + 2 * d[c] + uu[c] + 4) / 8; + md[c] = (5 * d[c] + 2 * u[c] + dd[c] + 4) / 8; + } + } + + dst_img.set_clipped(x, y - 1, mu); + dst_img.set_clipped(x, y, md); + + } // x + + } // y + + return true; +} + +static void xuastc_fixup_pvrtc1_4_modulation_rgb( + const basisu::vector2D& temp_image, + const uint32_t* pPVRTC_endpoints, + void* pDst_blocks, + uint32_t num_blocks_x, uint32_t num_blocks_y, bool from_alpha) +{ + const uint32_t x_mask = num_blocks_x - 1; + const uint32_t y_mask = num_blocks_y - 1; + const uint32_t x_bits = basisu::total_bits(x_mask); + const uint32_t y_bits = basisu::total_bits(y_mask); + const uint32_t min_bits = basisu::minimum(x_bits, y_bits); + //const uint32_t max_bits = basisu::maximum(x_bits, y_bits); + const uint32_t swizzle_mask = (1 << (min_bits * 2)) - 1; + + uint32_t block_index = 0; + + // really 3x3 + int e0[4][4], e1[4][4]; + + for (int y = 0; y < static_cast(num_blocks_y); y++) + { + const uint32_t* pE_rows[3]; + + for (int ey = 0; ey < 3; ey++) + { + int by = y + ey - 1; + + const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; + + pE_rows[ey] = pE; + + for (int ex = 0; ex < 3; ex++) + { + int bx = 0 + ex - 1; + + const uint32_t e = pE[bx & x_mask]; + + e0[ex][ey] = (get_opaque_endpoint_l0(e) * 255) / 31; + e1[ex][ey] = (get_opaque_endpoint_l1(e) * 255) / 31; + } + } + + const uint32_t y_swizzle = (g_pvrtc_swizzle_table[y >> 8] << 16) | g_pvrtc_swizzle_table[y & 0xFF]; + + for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) + { + color32 block_pixels[4][4]; + temp_image.extract_block_clamped(&block_pixels[0][0], x * 4, y * 4, 4, 4); + + if (from_alpha) + { + // Just set RGB to alpha to avoid adding complexity below. + for (uint32_t i = 0; i < 16; i++) + { + const uint8_t a = ((color32*)block_pixels)[i].a; + ((color32*)block_pixels)[i].set(a, a, a, 255); + } + } + + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); + + uint32_t swizzled = x_swizzle | y_swizzle; + if (num_blocks_x != num_blocks_y) + { + swizzled &= swizzle_mask; + + if (num_blocks_x > num_blocks_y) + swizzled |= ((x >> min_bits) << (min_bits * 2)); + else + swizzled |= ((y >> min_bits) << (min_bits * 2)); + } + + pvrtc4_block* pDst_block = static_cast(pDst_blocks) + swizzled; + pDst_block->m_endpoints = pPVRTC_endpoints[block_index]; + + { + const uint32_t ex = 2; + int bx = x + ex - 1; + bx &= x_mask; + +#define BUT_DO_ROW(ey) \ + { \ + const uint32_t e = pE_rows[ey][bx]; \ + e0[ex][ey] = (get_opaque_endpoint_l0(e) * 255) / 31; \ + e1[ex][ey] = (get_opaque_endpoint_l1(e) * 255) / 31; \ + } + + BUT_DO_ROW(0); + BUT_DO_ROW(1); + BUT_DO_ROW(2); +#undef BUT_DO_ROW + } + + uint32_t mod = 0; + +#define BUT_DO_PIX(lx, ly, w0, w1, w2, w3) \ + { \ + int ca_l = a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3; \ + int cb_l = b0 * w0 + b1 * w1 + b2 * w2 + b3 * w3; \ + int cl = (block_pixels[ly][lx].r + block_pixels[ly][lx].g + block_pixels[ly][lx].b) * 16; \ + int dl = cb_l - ca_l; \ + int vl = cl - ca_l; \ + int p = vl * 16; \ + if (ca_l > cb_l) { p = -p; dl = -dl; } \ + uint32_t m = 0; \ + if (p > 3 * dl) m = (uint32_t)(1 << ((ly) * 8 + (lx) * 2)); \ + if (p > 8 * dl) m = (uint32_t)(2 << ((ly) * 8 + (lx) * 2)); \ + if (p > 13 * dl) m = (uint32_t)(3 << ((ly) * 8 + (lx) * 2)); \ + mod |= m; \ + } + + { + const uint32_t ex = 0, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(0, 0, 4, 4, 4, 4); + BUT_DO_PIX(1, 0, 2, 6, 2, 6); + BUT_DO_PIX(0, 1, 2, 2, 6, 6); + BUT_DO_PIX(1, 1, 1, 3, 3, 9); + } + + { + const uint32_t ex = 1, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(2, 0, 8, 0, 8, 0); + BUT_DO_PIX(3, 0, 6, 2, 6, 2); + BUT_DO_PIX(2, 1, 4, 0, 12, 0); + BUT_DO_PIX(3, 1, 3, 1, 9, 3); + } + + { + const uint32_t ex = 0, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(0, 2, 8, 8, 0, 0); + BUT_DO_PIX(1, 2, 4, 12, 0, 0); + BUT_DO_PIX(0, 3, 6, 6, 2, 2); + BUT_DO_PIX(1, 3, 3, 9, 1, 3); + } + + { + const uint32_t ex = 1, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + BUT_DO_PIX(2, 2, 16, 0, 0, 0); + BUT_DO_PIX(3, 2, 12, 4, 0, 0); + BUT_DO_PIX(2, 3, 12, 0, 4, 0); + BUT_DO_PIX(3, 3, 9, 3, 3, 1); + } +#undef BUT_DO_PIX + + pDst_block->m_modulation = mod; + + e0[0][0] = e0[1][0]; e0[1][0] = e0[2][0]; + e0[0][1] = e0[1][1]; e0[1][1] = e0[2][1]; + e0[0][2] = e0[1][2]; e0[1][2] = e0[2][2]; + + e1[0][0] = e1[1][0]; e1[1][0] = e1[2][0]; + e1[0][1] = e1[1][1]; e1[1][1] = e1[2][1]; + e1[0][2] = e1[1][2]; e1[1][2] = e1[2][2]; + + } // x + } // y +} + +static void xuastc_fixup_pvrtc1_4_modulation_rgba( + const basisu::vector2D& temp_image, + const uint32_t* pPVRTC_endpoints, + void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y) +{ + const uint32_t x_mask = num_blocks_x - 1; + const uint32_t y_mask = num_blocks_y - 1; + const uint32_t x_bits = basisu::total_bits(x_mask); + const uint32_t y_bits = basisu::total_bits(y_mask); + const uint32_t min_bits = basisu::minimum(x_bits, y_bits); + //const uint32_t max_bits = basisu::maximum(x_bits, y_bits); + const uint32_t swizzle_mask = (1 << (min_bits * 2)) - 1; + + uint32_t block_index = 0; + + // really 3x3 + int e0[4][4], e1[4][4]; + + for (int y = 0; y < static_cast(num_blocks_y); y++) + { + const uint32_t* pE_rows[3]; + + for (int ey = 0; ey < 3; ey++) + { + int by = y + ey - 1; + + const uint32_t* pE = &pPVRTC_endpoints[(by & y_mask) * num_blocks_x]; + + pE_rows[ey] = pE; + + for (int ex = 0; ex < 3; ex++) + { + int bx = 0 + ex - 1; + + const uint32_t e = pE[bx & x_mask]; + + e0[ex][ey] = get_endpoint_l8(e, 0); + e1[ex][ey] = get_endpoint_l8(e, 1); + } + } + + const uint32_t y_swizzle = (g_pvrtc_swizzle_table[y >> 8] << 16) | g_pvrtc_swizzle_table[y & 0xFF]; + + for (int x = 0; x < static_cast(num_blocks_x); x++, block_index++) + { + color32 block_pixels[4][4]; + temp_image.extract_block_clamped(&block_pixels[0][0], x * 4, y * 4, 4, 4); + + const uint32_t x_swizzle = (g_pvrtc_swizzle_table[x >> 8] << 17) | (g_pvrtc_swizzle_table[x & 0xFF] << 1); + + uint32_t swizzled = x_swizzle | y_swizzle; + if (num_blocks_x != num_blocks_y) + { + swizzled &= swizzle_mask; + + if (num_blocks_x > num_blocks_y) + swizzled |= ((x >> min_bits) << (min_bits * 2)); + else + swizzled |= ((y >> min_bits) << (min_bits * 2)); + } + + pvrtc4_block* pDst_block = static_cast(pDst_blocks) + swizzled; + pDst_block->m_endpoints = pPVRTC_endpoints[block_index]; + + { + const uint32_t ex = 2; + int bx = x + ex - 1; + bx &= x_mask; + +#define DO_ROW(ey) \ + { \ + const uint32_t e = pE_rows[ey][bx]; \ + e0[ex][ey] = get_endpoint_l8(e, 0); \ + e1[ex][ey] = get_endpoint_l8(e, 1); \ + } + + DO_ROW(0); + DO_ROW(1); + DO_ROW(2); +#undef DO_ROW + } + + uint32_t mod = 0; + +#define DO_PIX(lx, ly, w0, w1, w2, w3) \ + { \ + int ca_l = a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3; \ + int cb_l = b0 * w0 + b1 * w1 + b2 * w2 + b3 * w3; \ + int cl = 16 * (block_pixels[ly][lx].r + block_pixels[ly][lx].g + block_pixels[ly][lx].b + block_pixels[ly][lx].a); \ + int dl = cb_l - ca_l; \ + int vl = cl - ca_l; \ + int p = vl * 16; \ + if (ca_l > cb_l) { p = -p; dl = -dl; } \ + uint32_t m = 0; \ + if (p > 3 * dl) m = (uint32_t)(1 << ((ly) * 8 + (lx) * 2)); \ + if (p > 8 * dl) m = (uint32_t)(2 << ((ly) * 8 + (lx) * 2)); \ + if (p > 13 * dl) m = (uint32_t)(3 << ((ly) * 8 + (lx) * 2)); \ + mod |= m; \ + } + + { + const uint32_t ex = 0, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(0, 0, 4, 4, 4, 4); + DO_PIX(1, 0, 2, 6, 2, 6); + DO_PIX(0, 1, 2, 2, 6, 6); + DO_PIX(1, 1, 1, 3, 3, 9); + } + + { + const uint32_t ex = 1, ey = 0; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(2, 0, 8, 0, 8, 0); + DO_PIX(3, 0, 6, 2, 6, 2); + DO_PIX(2, 1, 4, 0, 12, 0); + DO_PIX(3, 1, 3, 1, 9, 3); + } + + { + const uint32_t ex = 0, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(0, 2, 8, 8, 0, 0); + DO_PIX(1, 2, 4, 12, 0, 0); + DO_PIX(0, 3, 6, 6, 2, 2); + DO_PIX(1, 3, 3, 9, 1, 3); + } + + { + const uint32_t ex = 1, ey = 1; + const int a0 = e0[ex][ey], a1 = e0[ex + 1][ey], a2 = e0[ex][ey + 1], a3 = e0[ex + 1][ey + 1]; + const int b0 = e1[ex][ey], b1 = e1[ex + 1][ey], b2 = e1[ex][ey + 1], b3 = e1[ex + 1][ey + 1]; + DO_PIX(2, 2, 16, 0, 0, 0); + DO_PIX(3, 2, 12, 4, 0, 0); + DO_PIX(2, 3, 12, 0, 4, 0); + DO_PIX(3, 3, 9, 3, 3, 1); + } +#undef DO_PIX + + pDst_block->m_modulation = mod; + + e0[0][0] = e0[1][0]; e0[1][0] = e0[2][0]; + e0[0][1] = e0[1][1]; e0[1][1] = e0[2][1]; + e0[0][2] = e0[1][2]; e0[1][2] = e0[2][2]; + + e1[0][0] = e1[1][0]; e1[1][0] = e1[2][0]; + e1[0][1] = e1[1][1]; e1[1][1] = e1[2][1]; + e1[0][2] = e1[1][2]; e1[1][2] = e1[2][2]; + + } // x + } // y +} + +void encode_pvrtc1( + block_format fmt, void* pDst_blocks, + const basisu::vector2D &temp_image, + uint32_t dst_num_blocks_x, uint32_t dst_num_blocks_y, bool from_alpha) +{ + assert((fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA)); + + basisu::vector2D pvrtc1_endpoints(dst_num_blocks_x, dst_num_blocks_y); + + // Determine block endpoints + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + color32 block_pixels[4 * 4]; + + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + color32 low_color(255, 255, 255, 255), high_color(0, 0, 0, 0); + + for (uint32_t i = 0; i < 16; i++) + { + low_color = color32::comp_min(low_color, block_pixels[i]); + high_color = color32::comp_max(high_color, block_pixels[i]); + } + + if ((fmt == block_format::cPVRTC1_4_RGB) && (from_alpha)) + { + low_color.set(low_color.a, low_color.a, low_color.a, 255); + high_color.set(high_color.a, high_color.a, high_color.a, 255); + } + + pvrtc4_block temp; + if (fmt == block_format::cPVRTC1_4_RGBA) + { + temp.set_endpoint_floor(0, low_color); + temp.set_endpoint_ceil(1, high_color); + } + else + { + temp.set_opaque_endpoint_floor(0, low_color); + temp.set_opaque_endpoint_ceil(1, high_color); + } + + pvrtc1_endpoints(dst_bx, dst_by) = temp.m_endpoints; + } // dst_bx + + } // dst_by + + // Create PVRTC1 texture data. + if (fmt == block_format::cPVRTC1_4_RGBA) + xuastc_fixup_pvrtc1_4_modulation_rgba(temp_image, pvrtc1_endpoints.get_ptr(), pDst_blocks, dst_num_blocks_x, dst_num_blocks_y); + else + xuastc_fixup_pvrtc1_4_modulation_rgb(temp_image, pvrtc1_endpoints.get_ptr(), pDst_blocks, dst_num_blocks_x, dst_num_blocks_y, from_alpha); +} + +#endif // BASISD_SUPPORT_XUASTC + +static inline bool blocks_same_solid_colors(const astc_helpers::log_astc_block& a, const astc_helpers::log_astc_block& b, uint32_t tol) +{ + if ((!a.m_solid_color_flag_ldr) || (!b.m_solid_color_flag_ldr)) + return false; + + if (tol == 0) + { + return (a.m_solid_color[0] == b.m_solid_color[0]) && (a.m_solid_color[1] == b.m_solid_color[1]) && + (a.m_solid_color[2] == b.m_solid_color[2]) && (a.m_solid_color[3] == b.m_solid_color[3]); + } + + for (uint32_t i = 0; i < 4; i++) + { + int ac = a.m_solid_color[i] >> 8; + int bc = b.m_solid_color[i] >> 8; + + const int dl = basisu::iabs((int)ac - (int)bc); + if (dl > (int)tol) + return false; + } + + return true; +} + +static inline bool blocks_same_single_subset_endpoints(const astc_helpers::log_astc_block& a, const astc_helpers::log_astc_block& b, uint32_t tol) +{ + if (a.m_solid_color_flag_ldr || b.m_solid_color_flag_ldr) + return false; + + if (a.m_dual_plane || b.m_dual_plane) + return false; + + if ((a.m_num_partitions > 1) || (b.m_num_partitions > 1)) + return false; + + if (a.m_color_endpoint_modes[0] != b.m_color_endpoint_modes[0]) + return false; + + if (a.m_endpoint_ise_range != b.m_endpoint_ise_range) + return false; + + if (tol > 0) + { + // Compare endpoints with tolerance + color_rgba al, ah; + astc_ldr_t::decode_endpoints(a.m_color_endpoint_modes[0], a.m_endpoints, a.m_endpoint_ise_range, al, ah); + + color_rgba bl, bh; + astc_ldr_t::decode_endpoints(b.m_color_endpoint_modes[0], b.m_endpoints, b.m_endpoint_ise_range, bl, bh); + + for (uint32_t i = 0; i < 4; i++) + { + const int dl = basisu::iabs((int)al[i] - (int)bl[i]); + if (dl > (int)tol) + return false; + + const int dh = basisu::iabs((int)ah[i] - (int)bh[i]); + if (dh > (int)tol) + return false; + } + } + else + { + uint32_t total_endpoint_vals = astc_helpers::get_num_cem_values(a.m_color_endpoint_modes[0]); + if (memcmp(a.m_endpoints, b.m_endpoints, total_endpoint_vals) != 0) + return false; + } + + return true; +} + +static inline bool block_has_alpha(const astc_helpers::log_astc_block& a) +{ + if (a.m_solid_color_flag_ldr) + { + return (a.m_solid_color[3] >> 8) != 255; + } + + assert(a.m_num_partitions == 1); + + return astc_helpers::does_cem_have_alpha(a.m_color_endpoint_modes[0]); +} + +static void astc_upsample_grid_weights(const astc_helpers::log_astc_block& log_blk, uint8_t* pDst_weights, uint32_t block_width, uint32_t block_height) +{ + // Skip if solid (which is fine) + if (log_blk.m_solid_color_flag_ldr) + { +#if defined(DEBUG) || defined(_DEBUG) + memset(pDst_weights, 0xFF, block_width * block_height); +#endif + return; + } + + assert((log_blk.m_grid_width <= block_width) && (log_blk.m_grid_height <= block_height)); + + uint8_t dequantized_weights[astc_helpers::MAX_BLOCK_PIXELS]; + + const uint32_t total_weight_vals = log_blk.m_grid_width * log_blk.m_grid_height; + + const astc_helpers::dequant_table& weight_dequant_tab = astc_helpers::g_dequant_tables.get_weight_tab(log_blk.m_weight_ise_range); + const uint8_t* pWeight_dequant = weight_dequant_tab.m_ISE_to_val.data(); + + for (uint32_t i = 0; i < total_weight_vals; i++) + { + assert(log_blk.m_weights[i] < weight_dequant_tab.m_ISE_to_val.size_u32()); + + dequantized_weights[i] = pWeight_dequant[log_blk.m_weights[i]]; + } + + if ((log_blk.m_grid_width < block_width) || (log_blk.m_grid_height < block_height)) + { + astc_helpers::upsample_weight_grid_xuastc_ldr(block_width, block_height, log_blk.m_grid_width, log_blk.m_grid_height, dequantized_weights, pDst_weights, nullptr, nullptr); + } + else + { + memcpy(pDst_weights, dequantized_weights, block_width * block_height); + } +} + +bool basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, + void* pDst_blocks, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, + const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, + const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels, + basisu_transcoder_state* pState, uint32_t output_rows_in_pixels, int channel0, int channel1, uint32_t decode_flags) +{ + BASISU_NOTE_UNUSED(pState); + BASISU_NOTE_UNUSED(bc1_allow_threecolor_blocks); + +#if BASISD_SUPPORT_XUASTC + + assert(g_transcoder_initialized); + if (!g_transcoder_initialized) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: Transcoder not globally initialized.\n"); + return false; + } + + if (block_format_is_hdr(fmt)) + { + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: Invalid fmt argument\n"); + return false; + } + + //const uint32_t total_blocks = num_blocks_x * num_blocks_y; + + const uint32_t src_block_width = basis_tex_format_get_block_width(src_format), src_block_height = basis_tex_format_get_block_height(src_format); + + const uint32_t dst_fmt_block_width = get_block_width(fmt), dst_fmt_block_height = get_block_height(fmt); + const bool dst_fmt_is_astc = block_format_is_astc(fmt); + const bool dst_fmt_is_pvrtc1 = (fmt == block_format::cPVRTC1_4_RGB) || (fmt == block_format::cPVRTC1_4_RGBA); + + if (dst_fmt_is_pvrtc1) + { + if (!basisu::is_pow2(orig_width) || !basisu::is_pow2(orig_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: PVRTC1 requires power of 2 texture dimensions\n"); + return false; + } + } + + const bool is_uncompressed_fmt = basis_block_format_is_uncompressed(fmt); + if (!output_row_pitch_in_blocks_or_pixels) + { + if (is_uncompressed_fmt) + output_row_pitch_in_blocks_or_pixels = orig_width; + else + output_row_pitch_in_blocks_or_pixels = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + } + + if (is_uncompressed_fmt) + { + if (!output_rows_in_pixels) + output_rows_in_pixels = orig_height; + } + + const bool high_quality = (decode_flags & cDecodeFlagsHighQuality) != 0; + const bool enable_fast_bc7_transcoding = (decode_flags & cDecodeFlagXUASTCLDRDisableFastBC7Transcoding) == 0; + const bool from_alpha = has_alpha && (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; + const bool disable_deblocking = (decode_flags & cDecodeFlagsNoDeblockFiltering) != 0; + const bool stronger_deblocking = ((decode_flags & cDecodeFlagsStrongerDeblockFiltering) != 0) || ((src_block_width > 8) || (src_block_height > 8)); + const bool force_deblocking = (decode_flags & cDecodeFlagsForceDeblockFiltering) != 0; + const bool deblock_filtering = !disable_deblocking && (force_deblocking || ((src_block_width > 8) || (src_block_height > 6))); + + const uint32_t bc7f_flags = high_quality ? bc7f::cPackBC7FlagDefaultPartiallyAnalytical : bc7f::cPackBC7FlagDefault; + etc1f::pack_etc1_state etc1_pack_state; + + if (basis_tex_format_is_astc_ldr(src_format)) + { + // Plain ASTC LDR 4x4-12x12 - note it could be ANY valid/standard ASTC written by any ASTC encoder, so we cannot trust this ASTC data. + // It must be fully validated. + if (dst_fmt_is_astc) + { + assert(output_block_or_pixel_stride_in_bytes == sizeof(astc_helpers::astc_block)); + + if ((dst_fmt_block_width != src_block_width) || (dst_fmt_block_height != src_block_height)) + { + // ASTC block dimensions must match, i.e. we can't change the ASTC block size during transcoding. + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: fmt's ASTC block dimensions don't match the content's block dimensions\n"); + return false; + } + + // No transcoding needed, it's ASTC in->ASTC out. + memcpy(pDst_blocks, pImage_data, src_num_blocks_x * src_num_blocks_y * sizeof(astc_helpers::astc_block)); + } + else if (((src_block_width == 4) && (src_block_height == 4)) && (!dst_fmt_is_pvrtc1) && (!deblock_filtering)) + { + // Block dimensions aren't changing, no pvrtc1, no deblock filtering + if ((dst_fmt_block_width != 4) || (dst_fmt_block_height != 4)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: fmt's ASTC block dimensions don't match the content's block dimensions\n"); + return false; + } + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + + for (uint32_t block_y = 0; block_y < src_num_blocks_y; block_y++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + block_y * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t block_x = 0; block_x < src_num_blocks_x; block_x++) + { + color32 block_pixels[4 * 4]; + + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, 4, 4); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + // TODO: Specially handle solid block case + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, 4, 4, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + transcode_4x4_block( + fmt, + block_x, block_y, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + ++pSrc_phys_blk; + + } // block_x + + } // block_y + } + else if ((!deblock_filtering) && (!dst_fmt_is_pvrtc1)) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + // Compute how many source block rows we need to buffer so we have a multiple of 4 scanlines. The max # of scanlines is 20. + uint32_t num_src_block_rows_to_buffer = 1; + while ((num_src_block_rows_to_buffer * src_block_height) & 3) + num_src_block_rows_to_buffer++; + assert((num_src_block_rows_to_buffer >= 1) && (num_src_block_rows_to_buffer <= 4)); + + // Compute how many 4x4 dest blocks fit into these many source rows. + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + //const uint32_t num_dst_block_rows_to_buffer = (num_src_block_rows_to_buffer * src_block_height) >> 2; + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x * src_block_width, num_src_block_rows_to_buffer * src_block_height); + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + + for (uint32_t by = 0; by < src_num_blocks_y; by++) + { + const uint32_t buffered_src_block_row_y = (by % num_src_block_rows_to_buffer); + + for (uint32_t bx = 0; bx < src_num_blocks_x; bx++) + { + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, src_block_width, src_block_height); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, src_block_width, src_block_height, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &buffered_rows(bx * src_block_width, buffered_src_block_row_y * src_block_height); + + for (uint32_t y = 0; y < src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, src_block_width * sizeof(color32)); + + pSrc_pixels += src_block_width; + pDst_pixels += buffered_rows.get_width(); + } // y + + ++pSrc_phys_blk; + + } // block_x + + const bool final_src_block_row = (by == (src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + continue; + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / num_src_block_rows_to_buffer) * num_src_block_rows_to_buffer) * src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == num_src_block_rows_to_buffer) || (final_src_block_row)); + + // The maximum number of valid buffer scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * src_block_height); + assert(override_buffer_height); + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * src_block_height + 3) >> 2; + + for (uint32_t dst_ofs_by = 0; dst_ofs_by < total_dst_block_rows_to_emit; dst_ofs_by++) + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dst_ofs_by; + if (dst_by >= dst_num_blocks_y) + break; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + color32 block_pixels[4 * 4]; + + // Extract the 4x4 block pixels from our buffered rows, taking into account the actual # of valid scanlines inside the buffer. + buffered_rows.extract_block_clamped(block_pixels, dst_bx * 4, dst_ofs_by * 4, 4, 4, override_buffer_height); + + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + (dst_by * output_row_pitch_in_blocks_or_pixels + dst_bx) * output_block_or_pixel_stride_in_bytes; + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + } // dst_bx + + } // dst_ofs_by + + } // block_y + + } + else + { + // unpack entire 32bpp image into memory (needed for deblocking and PVRTC1) + // TODO: Add more memory efficient non-deblocking code path + basisu::vector2D temp_image; + + const uint32_t actual_width = src_block_width * src_num_blocks_x, actual_height = src_block_height * src_num_blocks_y; + + if (!temp_image.try_resize(actual_width, actual_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + + const astc_helpers::astc_block* pSrc_phys_blk = (const astc_helpers::astc_block*)pImage_data; + + astc_helpers::log_astc_block log_blk; + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t src_by = 0; src_by < src_num_blocks_y; src_by++) + { + const uint32_t img_y = src_by * src_block_height; + + for (uint32_t src_bx = 0; src_bx < src_num_blocks_x; src_bx++) + { + bool unpack_status = astc_helpers::unpack_block(pSrc_phys_blk, log_blk, src_block_width, src_block_height); + if (!unpack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::unpack_block() failed\n"); + return false; + } + + bool decode_status = astc_helpers::decode_block(log_blk, block_pixels, src_block_width, src_block_height, use_astc_srgb_decode_profile ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + color32* pSrc_pixels = (color32*)block_pixels; + color32* pDst_pixels = &temp_image(src_bx * src_block_width, img_y); + + for (uint32_t y = 0; y < src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, src_block_width * sizeof(color32)); + + pSrc_pixels += src_block_width; + pDst_pixels += temp_image.get_width(); + } // y + + ++pSrc_phys_blk; + + } // src_bx + } // src_by + + if (deblock_filtering) + { + if (!xuastc_deblock_filter( + src_block_width, src_block_height, + temp_image, temp_image, + stronger_deblocking, XUASTC_LDR_DEBLOCK_SKIP_THRESH)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + } + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + if (dst_fmt_is_pvrtc1) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + encode_pvrtc1(fmt, pDst_blocks, temp_image, dst_num_blocks_x, dst_num_blocks_y, from_alpha); + } + else + { + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + dst_by * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + + } // dst_bx + + } // dst_by + + } // if (dst_fmt_is_pvrtc1) + } + } + else + { + // Supercompressed XUASTC LDR 4x4-12x12 + // note use_astc_srgb_decode_profile can be ignored here, we'll use the decoded sRGB profile bit from the compressed stream. + + if (dst_fmt_is_astc) + { + // src and dst are ASTC - ideal case, just pack physical ASTC blocks to output buffer during transcoding. + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_dst_format_block_width; + uint32_t m_dst_format_block_height; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_dst_format_block_width = dst_fmt_block_width; + dec_state.m_dst_format_block_height = dst_fmt_block_height; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + BASISU_NOTE_UNUSED(has_alpha); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_dst_format_block_width) || (block_height != state.m_dst_format_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (1)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (2)\n"); + return false; + } + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + astc_helpers::astc_block* pDst_astc_block = (astc_helpers::astc_block*)((uint8_t*)state.m_pDst_blocks + (by * state.m_output_row_pitch_in_blocks_or_pixels + bx) * state.m_output_block_or_pixel_stride_in_bytes); + + bool pack_status = astc_helpers::pack_astc_block(*pDst_astc_block, log_blk); + if (!pack_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::pack_astc_block() failed\n"); + return false; + } + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + else if ((fmt == block_format::cBC7) && (src_block_width == 8) && (src_block_height == 6) && + (enable_fast_bc7_transcoding) && (!high_quality) && (!deblock_filtering)) + { + // src is ASTC LDR 8x6, destination is BC7, no deblocking: buffer 2 rows of ASTC logical blocks, favor fast pure transcode to BC7 whenever possible. + // transcodes 2 ASTC 8x6 blocks (a tile of 1x2 or 2*48=96 pixels) to 6 BC7 blocks (a tile of 2x3 of 6*16=96 pixels) + // no BC7 block crosses more than 2 ASTC blocks making this easy if the source blocks are only solid or 1 subset + + const uint32_t num_src_block_rows_to_buffer = 2; + + assert(((num_src_block_rows_to_buffer* src_block_height) & 3) == 0); + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x, num_src_block_rows_to_buffer); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D* m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + uint32_t m_total_src_blocks_unpacked; + uint32_t m_total_src_blocks_partial_unpacked; + uint32_t m_total_blocks_transcoded; + uint32_t m_total_blocks_encoded; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_total_src_blocks_unpacked = 0; + dec_state.m_total_src_blocks_partial_unpacked = 0; + dec_state.m_total_blocks_transcoded = 0; + dec_state.m_total_blocks_encoded = 0; + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + assert(state.m_num_src_block_rows_to_buffer == 2); // hardcoded for 6x6 + const astc_helpers::decode_mode astc_dec_mode = state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t buffered_src_block_row_y = (by & 1); + + memcpy(&(*state.m_pBuffered_rows)(bx, buffered_src_block_row_y), &log_blk, sizeof(log_blk)); + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + return true; + + // We have a full 1-2 rows of ASTC 6x6 blocks to process to BC7. + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + +#if defined(DEBUG) || defined(_DEBUG) + // The maximum number of valid buffer pixel scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); +#endif + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + color_rgba unpacked_src_blocks[2][8 * 6]; // [astc_by][pixel] + uint8_t upsampled_src_weights[2][8 * 6]; // [astc_by][pixel] + color_rgba temp_pixels_16[16]; + + // Process each source ASTC 8x6 block group, 1x2 ASTC blocks at a time + for (uint32_t src_bx = 0; src_bx < state.m_src_num_blocks_x; src_bx++) + { + bool has_unpacked_src_blocks[2] = { }; // [astc_by] + + // Grab pointers to the 1x2 src ASTC blocks we'll be transcoding + const astc_helpers::log_astc_block* pSrc_log_blocks[2]; // [astc_by] + + pSrc_log_blocks[0] = &state.m_pBuffered_rows->at(src_bx, 0); + pSrc_log_blocks[1] = &state.m_pBuffered_rows->at(src_bx, basisu::minimum(1, num_buffered_src_block_rows - 1)); + + // From here we can always assume 2x2 src ASTC 6x6 blocks, with ASTC block pointers duplicated at the borders if needed. + + const astc_helpers::log_astc_block* pU = pSrc_log_blocks[0]; + const astc_helpers::log_astc_block* pL = pSrc_log_blocks[1]; + + if (blocks_same_solid_colors(*pU, *pL, 0)) + { + // Easy and fast case: All 2 ASTC blocks are solid and the same color, so all 6 BC7 blocks are solid too and the same color. + color_rgba sc; + sc.r = (uint8_t)(pU->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pU->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pU->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pU->m_solid_color[3] >> 8); + + bc7_block temp_blk; + bc7f::pack_mode5_solid((uint8_t*)&temp_blk, sc); + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + memcpy(pDst_block_u8, &temp_blk, sizeof(bc7_block)); + } // dx + } // dy + + continue; + } + + uint32_t num_hard_src_blocks = 0; + + for (uint32_t y = 0; y < 2; y++) + num_hard_src_blocks += (pSrc_log_blocks[y]->m_dual_plane || (pSrc_log_blocks[y]->m_num_partitions > 1)); + + if (num_hard_src_blocks == 2) + { + // All src blocks hard, no easy optimizations, so unpack and encode pixels analytically + for (uint32_t y = 0; y < 2; y++) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[y], &unpacked_src_blocks[y][0], 8, 6, astc_dec_mode); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + } + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + + const uint32_t sy_div6 = sy / 6; + const uint32_t src_row_ofs = (sy % 6) * 8; + + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sy_div6][sx + src_row_ofs]; + } // x + } // y + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } // dx + } // dy + + continue; + } + + // One or both of the 2 source blocks is solid or 1 subset. + + // For simplicity: First unpack the first plane weight grids (unless solid) + for (uint32_t y = 0; y < 2; y++) + astc_upsample_grid_weights(*pSrc_log_blocks[y], &upsampled_src_weights[y][0], 8, 6); + + const uint32_t ENDPOINT_TOL = 0; + + // Process each of the up to 6 destination BC7 blocks. + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 2; dx++) + { + const uint32_t dst_bx = (src_bx << 1) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // BC7 block only overlaps 1 or 2 ASTC blocks. + const int top_dy = dy * 4; + const int bot_dy = top_dy + 3; + + const int top_by = top_dy / 6; + const int bot_by = bot_dy / 6; + + const astc_helpers::log_astc_block* pB0 = pSrc_log_blocks[top_by]; + const astc_helpers::log_astc_block* pB1 = pSrc_log_blocks[bot_by]; + + const bool single_src_block = (top_by == bot_by); + + bool full_encode_flag = false; + + if (pB0->m_dual_plane || pB1->m_dual_plane || + (pB0->m_num_partitions > 1) || (pB1->m_num_partitions > 1)) + { + // Either block is complex, fall back to reencoding + full_encode_flag = true; + } + else if (single_src_block) + { + assert(pB0 == pB1); + + // BC7 block is at a corner, and only overlaps a single ASTC block - output solid or single subset BC7 + if (pB0->m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + } + else + { + // Output mode 6 BC7 + bc7f::pack_from_astc_single_subset(pDst_block_u8, *pB0, &upsampled_src_weights[top_by][0], dx * 4, (dy * 4) % 6, 8, 6); + } + + state.m_total_blocks_transcoded++; + } + // must be overlapping 2 ASTC blocks, can't be both solid as we've already checked + else if (blocks_same_single_subset_endpoints(*pB0, *pB1, ENDPOINT_TOL)) + { + // BC7 blocks overlaps 2 single subset ASTC blocks, both have the same or very similar endpoints, output mode 6 BC7 + bc7f::pack_from_astc_to_single_subset_same_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_by][0], + *pB1, &upsampled_src_weights[bot_by][0], + dx, dy, + 8, 6); + + state.m_total_blocks_transcoded++; + } + else if (!block_has_alpha(*pB0) && !block_has_alpha(*pB1)) + { + bool fallback_encode_flag = false; + + // BC7 block overlaps 2 ASTC blocks with different endpoints (or solid colors) - output 2 subset mode 1 BC7 + if (!bc7f::pack_from_astc_8x6_to_two_subsets_different_endpoints_hq( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_by][0], + *pB1, &upsampled_src_weights[bot_by][0], + dx, dy, state.m_used_srgb_astc_decode_mode, fallback_encode_flag)) + { + full_encode_flag = true; + } + else + { + if (fallback_encode_flag) + state.m_total_src_blocks_partial_unpacked++; + else + state.m_total_blocks_transcoded++; + } + } + else + { + full_encode_flag = true; + } + + if (full_encode_flag) + { + // one or both ASTC blocks are just too complex, unpack and reencode + if (!has_unpacked_src_blocks[top_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB0, &unpacked_src_blocks[top_by][0], 8, 6, astc_dec_mode, &upsampled_src_weights[top_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[top_by] = true; + } + + if (!has_unpacked_src_blocks[bot_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB1, &unpacked_src_blocks[bot_by][0], 8, 6, astc_dec_mode, &upsampled_src_weights[bot_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[bot_by] = true; + } + + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + + assert(has_unpacked_src_blocks[sy / 6]); + + const uint32_t sy_div6 = sy / 6; + const uint32_t src_row_ofs = (sy % 6) * 8; + + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sy_div6][sx + src_row_ofs]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // dx + } // dy + + } // src_bx + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (basisu::g_debug_printf) + { + basisu::fmt_debug_printf("Total src blocks: {}, Total src blocks fully unpacked to pixels: {}\n", + dec_state.m_src_num_blocks_x * dec_state.m_src_num_blocks_y, + dec_state.m_total_src_blocks_unpacked); + + basisu::fmt_debug_printf("Total dst blocks: {}, Total blocks transcoded: {}, fully encoded: {}, total partially unpacked/fast mode 6 encoded: {}\n", + dec_state.m_dst_num_blocks_x * dec_state.m_dst_num_blocks_y, + dec_state.m_total_blocks_transcoded, dec_state.m_total_blocks_encoded, + dec_state.m_total_src_blocks_partial_unpacked); + } + + // end of 8x6->4x4 transcoder + } + else if ((fmt == block_format::cBC7) && (src_block_width == 6) && (src_block_height == 6) && + (enable_fast_bc7_transcoding) && (!high_quality) && (!deblock_filtering)) + { + // src is ASTC LDR 6x6, destination is BC7, no deblocking: buffer 2 rows of ASTC logical blocks, favor fast pure transcode to BC7 whenever possible. + // This path is maddenningly tricky, but the speed gains in certain use cases are worth it. + + const uint32_t num_src_block_rows_to_buffer = 2; + + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x, num_src_block_rows_to_buffer); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D *m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + uint32_t m_total_src_blocks_unpacked; + uint32_t m_total_src_blocks_partial_unpacked; + uint32_t m_total_blocks_transcoded; + uint32_t m_total_blocks_encoded; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_total_src_blocks_unpacked = 0; + dec_state.m_total_src_blocks_partial_unpacked = 0; + dec_state.m_total_blocks_transcoded = 0; + dec_state.m_total_blocks_encoded = 0; + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + assert(state.m_num_src_block_rows_to_buffer == 2); // hardcoded for 6x6 + const astc_helpers::decode_mode astc_dec_mode = state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8; + + const uint32_t buffered_src_block_row_y = (by & 1); + + memcpy(&(*state.m_pBuffered_rows)(bx, buffered_src_block_row_y), &log_blk, sizeof(log_blk)); + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ((buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row)) + return true; + + // We have a full 1-2 rows of ASTC 6x6 blocks to process to BC7. + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + +#if defined(DEBUG) || defined(_DEBUG) + // The maximum number of valid buffer pixel scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); +#endif + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + color_rgba unpacked_src_blocks[2][2][6 * 6]; // [x][y][pixel] + uint8_t upsampled_src_weights[2][2][6 * 6]; // [x][y][pixel] + color_rgba temp_pixels_16[16]; + + // Process each source ASTC 6x6 block group, 2x2 ASTC blocks at a time (12x12 pixels, or 3x3 BC7 blocks) + for (uint32_t src_bx = 0; src_bx < state.m_src_num_blocks_x; src_bx += 2) + { + bool has_unpacked_src_blocks[2][2] = { }; // [x][y] + + // Grab pointers to the 2x2 src ASTC blocks we'll be transcoding + const astc_helpers::log_astc_block *pSrc_log_blocks[2][2]; // [x][y] + + pSrc_log_blocks[0][0] = &state.m_pBuffered_rows->at(src_bx, 0); + pSrc_log_blocks[1][0] = &state.m_pBuffered_rows->at(basisu::minimum(src_bx + 1, state.m_src_num_blocks_x - 1), 0); + + pSrc_log_blocks[0][1] = &state.m_pBuffered_rows->at(src_bx, basisu::minimum(1, num_buffered_src_block_rows - 1)); + + pSrc_log_blocks[1][1] = &state.m_pBuffered_rows->at( + basisu::minimum(src_bx + 1, state.m_src_num_blocks_x - 1), + basisu::minimum(1, num_buffered_src_block_rows - 1)); + + // From here we can always assume 2x2 src ASTC 6x6 blocks, with ASTC block pointers duplicated at the borders if needed. + + const astc_helpers::log_astc_block* pUL = pSrc_log_blocks[0][0]; + + // First see if all the astc blocks are the same solid color. Likely a common case on some images. + bool all_solid = true; + for (uint32_t y = 0; (y < 2) && all_solid; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + if (!pSrc_log_blocks[x][y]->m_solid_color_flag_ldr) + { + all_solid = false; + break; + } + + if ((pSrc_log_blocks[x][y]->m_solid_color[0] != pUL->m_solid_color[0]) || (pSrc_log_blocks[x][y]->m_solid_color[1] != pUL->m_solid_color[1]) || + (pSrc_log_blocks[x][y]->m_solid_color[2] != pUL->m_solid_color[2]) || (pSrc_log_blocks[x][y]->m_solid_color[3] != pUL->m_solid_color[3])) + { + all_solid = false; + break; + } + } + } + + if (all_solid) + { + // Easy and fast case: All 4 ASTC blocks are solid and the same color, so all 9 BC7 blocks are solid too and the same color. + color_rgba sc; + sc.r = (uint8_t)(pUL->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pUL->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pUL->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pUL->m_solid_color[3] >> 8); + + bc7_block temp_blk; + bc7f::pack_mode5_solid((uint8_t *)&temp_blk, sc); + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + memcpy(pDst_block_u8, &temp_blk, sizeof(bc7_block)); + } + } + + continue; + } + + uint32_t num_hard_src_blocks = 0; + + for (uint32_t y = 0; y < 2; y++) + for (uint32_t x = 0; x < 2; x++) + num_hard_src_blocks += (pSrc_log_blocks[x][y]->m_dual_plane || (pSrc_log_blocks[x][y]->m_num_partitions > 1)); + + if (num_hard_src_blocks == 4) + { + // All src blocks hard, no easy optimizations, so unpack and encode pixels analytically + for (uint32_t y = 0; y < 2; y++) + { + for (uint32_t x = 0; x < 2; x++) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[x][y], &unpacked_src_blocks[x][y][0], 6, 6, astc_dec_mode); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + } + } + + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + // TODO: Optimize + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } // dx + } // dy + + continue; + } + + // One or more of the 4 source blocks is solid or 1 subset. + + // For simplicity: First unpack the first plane weight grids (unless solid) + for (uint32_t y = 0; y < 2; y++) + for (uint32_t x = 0; x < 2; x++) + astc_upsample_grid_weights(*pSrc_log_blocks[x][y], &upsampled_src_weights[x][y][0], 6, 6); + + //const uint32_t ENDPOINT_TOL = 3; + const uint32_t ENDPOINT_TOL = 1; + + // Process each of the 9 destination BC7 blocks. + for (uint32_t dy = 0; dy < total_dst_block_rows_to_emit; dy++) // up to 3 dst block rows + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dx = 0; dx < 3; dx++) + { + // skip the central block, which we'll processed last + if ((dx == 1) && (dy == 1)) + continue; + + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + if (dst_bx >= state.m_dst_num_blocks_x) + break; + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // We're NOT at the center BC7 dst block. + // BC7 block only overlaps 1 or 2 ASTC blocks. + const int top_dx = dx * 4, top_dy = dy * 4; + const int bot_dx = top_dx + 3, bot_dy = top_dy + 3; + + const int top_bx = top_dx / 6, top_by = top_dy / 6; + const int bot_bx = bot_dx / 6, bot_by = bot_dy / 6; + + const astc_helpers::log_astc_block* pB0 = pSrc_log_blocks[top_bx][top_by]; + const astc_helpers::log_astc_block* pB1 = pSrc_log_blocks[bot_bx][bot_by]; + + // Note: because of row/col duplication at the edges of images, pB0 could equal pB1 even though we're not at a BC7 corner block. + + const bool single_src_block = (top_bx == bot_bx) && (top_by == bot_by); + + bool full_encode_flag = false; + + if (pB0->m_dual_plane || pB1->m_dual_plane || + (pB0->m_num_partitions > 1) || (pB1->m_num_partitions > 1)) + { + // Either block is complex, fall back to reencoding + full_encode_flag = true; + } + else if (single_src_block) + { + assert(pB0 == pB1); + + // BC7 block is at a corner, and only overlaps a single ASTC block - output solid or single subset BC7 + if (pB0->m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + } + else + { + // Output mode 6 BC7 + bc7f::pack_from_astc_single_subset(pDst_block_u8, *pB0, &upsampled_src_weights[top_bx][top_by][0], (dx * 4) % 6, (dy * 4) % 6, 6, 6); + } + + state.m_total_blocks_transcoded++; + } + // below here BC7 block always overlaps 2 ASTC 6x6 blocks (1 block case just ruled out) + else if (blocks_same_solid_colors(*pB0, *pB1, 0)) + { + // BC7 block overlaps 2 ASTC blocks, both solid colors, both same colors + + color_rgba sc; + sc.r = (uint8_t)(pB0->m_solid_color[0] >> 8); + sc.g = (uint8_t)(pB0->m_solid_color[1] >> 8); + sc.b = (uint8_t)(pB0->m_solid_color[2] >> 8); + sc.a = (uint8_t)(pB0->m_solid_color[3] >> 8); + bc7f::pack_mode5_solid(pDst_block_u8, sc); + + state.m_total_blocks_transcoded++; + } + else if (blocks_same_single_subset_endpoints(*pB0, *pB1, ENDPOINT_TOL)) + { + // BC7 blocks overlaps 2 single subset ASTC blocks, both have the same or very similar endpoints, output mode 6 BC7 + bc7f::pack_from_astc_to_single_subset_same_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_bx][top_by][0], + *pB1, &upsampled_src_weights[bot_bx][bot_by][0], + dx, dy, + 6, 6); + + state.m_total_blocks_transcoded++; + } + else if (!block_has_alpha(*pB0) && !block_has_alpha(*pB1)) + { + // BC7 block overlaps 2 ASTC blocks with different endpoints (or solid colors) - output 2 subset mode 1 BC7 + if (!bc7f::pack_from_astc_6x6_to_two_subsets_different_endpoints( + pDst_block_u8, + *pB0, &upsampled_src_weights[top_bx][top_by][0], + *pB1, &upsampled_src_weights[bot_bx][bot_by][0], + dx, dy)) + { + full_encode_flag = true; + } + else + { + state.m_total_blocks_transcoded++; + } + } + else + { + full_encode_flag = true; + } + + if (full_encode_flag) + { + // one or both ASTC blocks are just too complex, unpack and reencode + if (!has_unpacked_src_blocks[top_bx][top_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB0, &unpacked_src_blocks[top_bx][top_by][0], 6, 6, astc_dec_mode, &upsampled_src_weights[top_bx][top_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[top_bx][top_by] = true; + } + + if (!has_unpacked_src_blocks[bot_bx][bot_by]) + { + bool status = astc_helpers::decode_block_xuastc_ldr(*pB1, &unpacked_src_blocks[bot_bx][bot_by][0], 6, 6, astc_dec_mode, &upsampled_src_weights[bot_bx][bot_by][0]); + if (!status) + { + return false; + } + state.m_total_src_blocks_unpacked++; + has_unpacked_src_blocks[bot_bx][bot_by] = true; + } + + // TODO: Optimize + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + assert(has_unpacked_src_blocks[sx / 6][sy / 6]); + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // dx + } // dy + + // Now handle the center BC7 block - by this point we may have already decoded 1 or more ASTC 6x6 blocks. + const uint32_t dx = 1, dy = 1; + const uint32_t dst_bx = ((src_bx * 6) >> 2) + dx; + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dy; + + if ((dst_bx < state.m_dst_num_blocks_x) && (dst_by < state.m_dst_num_blocks_y)) + { + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + bool skip_full_encode = false; + + // the unfortunate middle BC7 block, overlaps all 4 ASTC blocks + // The 4 ASTC blocks cannot be all solid, and cannot be all hard. + if (num_hard_src_blocks == 0) + { + // all blocks are simple, see if we can find a simple case to handle quickly + bool top_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][0], *pSrc_log_blocks[1][0], 1); + bool bot_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][1], *pSrc_log_blocks[1][1], 1); + + bool left_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[0][0], *pSrc_log_blocks[0][1], 1); + bool right_same_solid_color = blocks_same_solid_colors(*pSrc_log_blocks[1][0], *pSrc_log_blocks[1][1], 1); + + bool top_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][0], *pSrc_log_blocks[1][0], ENDPOINT_TOL); + bool bot_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][1], *pSrc_log_blocks[1][1], ENDPOINT_TOL); + + bool left_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[0][0], *pSrc_log_blocks[0][1], ENDPOINT_TOL); + bool right_same_endpoints = blocks_same_single_subset_endpoints(*pSrc_log_blocks[1][0], *pSrc_log_blocks[1][1], ENDPOINT_TOL); + + bool top_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][0]) && !block_has_alpha(*pSrc_log_blocks[1][0]); + bool bot_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][1]) && !block_has_alpha(*pSrc_log_blocks[1][1]); + + bool left_no_alpha = !block_has_alpha(*pSrc_log_blocks[0][0]) && !block_has_alpha(*pSrc_log_blocks[0][1]); + bool right_no_alpha = !block_has_alpha(*pSrc_log_blocks[1][0]) && !block_has_alpha(*pSrc_log_blocks[1][1]); + + bool top_transcodable = (top_same_solid_color || top_same_endpoints) && top_no_alpha; + bool bot_transcodable = (bot_same_solid_color || bot_same_endpoints) && bot_no_alpha; + + bool left_transcodable = (left_same_solid_color || left_same_endpoints) && left_no_alpha; + bool right_transcodable = (right_same_solid_color || right_same_endpoints) && right_no_alpha; + + if (top_transcodable && bot_transcodable) + { + // BC7 mode 1 + bc7f::pack_astc_6x6_to_two_subsets_middle_block( + pDst_block_u8, + pSrc_log_blocks, upsampled_src_weights, + false); + state.m_total_blocks_transcoded++; + skip_full_encode = true; + } + else if (left_transcodable && right_transcodable) + { + // BC7 mode 1 + bc7f::pack_astc_6x6_to_two_subsets_middle_block( + pDst_block_u8, + pSrc_log_blocks, upsampled_src_weights, + true); + state.m_total_blocks_transcoded++; + skip_full_encode = true; + } + + // TODO: Handle non-exact cases + } + + if (!skip_full_encode) + { + // Center BC7 block scenario is complex, requires full decode+analytical BC7 encode. + if (has_unpacked_src_blocks[0][0] && has_unpacked_src_blocks[1][0] && has_unpacked_src_blocks[0][1] && has_unpacked_src_blocks[1][1]) + { + // We've already decoded all the ASTC 6x6 blocks fully, so grab central pixels + for (uint32_t y = 0; y < 4; y++) + { + const uint32_t sy = dy * 4 + y; + for (uint32_t x = 0; x < 4; x++) + { + const uint32_t sx = dx * 4 + x; + + temp_pixels_16[x + y * 4] = unpacked_src_blocks[sx / 6][sy / 6][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + else + { + for (uint32_t iby = 0; iby < 2; iby++) + { + for (uint32_t ibx = 0; ibx < 2; ibx++) + { + // Do a partial decode of the ASTC block, just to get those central pixels (big savings) + uint32_t start_x, start_y, end_x, end_y; + + switch (ibx + iby * 2) + { + case 0: start_x = 4; end_x = 6; start_y = 4; end_y = 6; break; + case 1: start_x = 0; end_x = 2; start_y = 4; end_y = 6; break; + case 2: start_x = 4; end_x = 6; start_y = 0; end_y = 2; break; + default: + case 3: start_x = 0; end_x = 2; start_y = 0; end_y = 2; break; + } + + // See if we've already decoded the block + if (has_unpacked_src_blocks[ibx][iby]) + { + for (uint32_t py = 0; py < 2; py++) + { + const uint32_t sy = start_y + py; + + for (uint32_t px = 0; px < 2; px++) + { + const uint32_t sx = start_x + px; + + temp_pixels_16[(px + ibx * 2) + (py + iby * 2) * 4] = unpacked_src_blocks[ibx][iby][(sx % 6) + (sy % 6) * 6]; + } // x + } // y + } + else + { + // Partial decode + color_rgba temp_pixels[6 * 6]; + + bool status = astc_helpers::decode_block_xuastc_ldr(*pSrc_log_blocks[ibx][iby], temp_pixels, 6, 6, astc_dec_mode, + &upsampled_src_weights[ibx][iby][0], start_x, start_y, end_x, end_y); + if (!status) + { + return false; + } + state.m_total_src_blocks_partial_unpacked++; + + for (uint32_t py = 0; py < 2; py++) + { + const uint32_t ey = py + iby * 2; + + for (uint32_t px = 0; px < 2; px++) + { + const uint32_t ex = px + ibx * 2; + + temp_pixels_16[ex + ey * 4] = temp_pixels[(start_x + px) + (start_y + py) * 6]; + } // x + } // y + } + } // x + } // y + + if (state.m_has_alpha) + bc7f::fast_pack_bc7_auto_rgba(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + else + bc7f::fast_pack_bc7_auto_rgb(pDst_block_u8, temp_pixels_16, state.m_bc7f_flags); + + state.m_total_blocks_encoded++; + } + + } // skip_full_encode + + } // if ((dst_bx < state.m_dst_num_blocks_x) && (dst_by < state.m_dst_num_blocks_y)) + + } // src_bx + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (basisu::g_debug_printf) + { + basisu::fmt_debug_printf("Total src blocks: {}, Total src blocks unpacked to pixels: {}, total partial unpacks: {}\n", + dec_state.m_src_num_blocks_x * dec_state.m_src_num_blocks_y, + dec_state.m_total_src_blocks_unpacked, dec_state.m_total_src_blocks_partial_unpacked); + + basisu::fmt_debug_printf("Total dst blocks: {}, Total blocks transcoded: {}, encoded: {}\n", + dec_state.m_dst_num_blocks_x * dec_state.m_dst_num_blocks_y, + dec_state.m_total_blocks_transcoded, dec_state.m_total_blocks_encoded); + } + } + else if (((src_block_width == 4) && (src_block_height == 4)) && (!dst_fmt_is_pvrtc1) && (!deblock_filtering)) + { + // src is ASTC LDR 4x4, destination block size must be 4x4, no PVRTC1, no deblocking. Directly pack to target format during transcoding. + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + block_format m_fmt; + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + int m_channel0, m_channel1; + bool m_high_quality; + bool m_enable_fast_bc7_transcoding; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + dec_state.m_fmt = fmt; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_enable_fast_bc7_transcoding = enable_fast_bc7_transcoding; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != 4) || (block_height != 4)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (by * state.m_output_row_pitch_in_blocks_or_pixels + bx) * state.m_output_block_or_pixel_stride_in_bytes; + + // Special fast cases for BC7 transcode target given common ASTC configs + if (state.m_fmt == block_format::cBC7) + { + if (log_blk.m_solid_color_flag_ldr) + { + color_rgba sc; + sc.r = (uint8_t)(log_blk.m_solid_color[0] >> 8); + sc.g = (uint8_t)(log_blk.m_solid_color[1] >> 8); + sc.b = (uint8_t)(log_blk.m_solid_color[2] >> 8); + sc.a = (uint8_t)(log_blk.m_solid_color[3] >> 8); + + bc7f::pack_mode5_solid(pDst_block_u8, sc); + return true; + } + else if (!log_blk.m_dual_plane && (log_blk.m_num_partitions == 1) && !state.m_high_quality && state.m_enable_fast_bc7_transcoding) + { + // TODO: This does cost a tiny amount of PSNR (.1-25 dB or so), but is way faster. + bc7f::pack_from_astc_4x4_single_subset(pDst_block_u8, log_blk); + return true; + } + } + + // Fall back to block pixel unpack then analytical encode. + color32 block_pixels[4 * 4]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, 4, 4, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + color32 alt_block_pixels[4 * 4]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, 4, 4, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < 16; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + transcode_4x4_block( + state.m_fmt, + bx, by, + state.m_pDst_blocks, pDst_block_u8, + block_pixels, + state.m_output_block_or_pixel_stride_in_bytes, state.m_output_row_pitch_in_blocks_or_pixels, state.m_output_rows_in_pixels, + state.m_channel0, state.m_channel1, + state.m_high_quality, state.m_from_alpha, + state.m_bc7f_flags, + *state.m_pEtc1_pack_state, + state.m_has_alpha ? 1 : 0); + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + else if ((deblock_filtering) || (dst_fmt_is_pvrtc1)) + { + // Completely general case. Unpack entire 32bpp image into memory (needed for deblocking and PVRTC1). + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + basisu::vector2D temp_image; + + if (!temp_image.try_resize(src_num_blocks_x * src_block_width, src_num_blocks_y * src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + + struct decode_state + { + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + basisu::vector2D* m_pTemp_image; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + }; + + decode_state dec_state; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + dec_state.m_pTemp_image = &temp_image; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + // sanity check vs. our vanilla/full-featured ASTC decoder + color32 alt_block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < state.m_src_block_width * state.m_src_block_height; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &(*state.m_pTemp_image)(bx * state.m_src_block_width, by * state.m_src_block_height); + + for (uint32_t y = 0; y < state.m_src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, state.m_src_block_width * sizeof(color32)); + + pSrc_pixels += state.m_src_block_width; + pDst_pixels += state.m_pTemp_image->get_width(); + } // y + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + + if (deblock_filtering) + { + if (!xuastc_deblock_filter( + decoded_image.m_actual_block_width, decoded_image.m_actual_block_height, + temp_image, temp_image, + stronger_deblocking, XUASTC_LDR_DEBLOCK_SKIP_THRESH)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: out of memory\n"); + return false; + } + } + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + if (dst_fmt_is_pvrtc1) + { + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + encode_pvrtc1(fmt, pDst_blocks, temp_image, dst_num_blocks_x, dst_num_blocks_y, from_alpha); + } + else + { + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + + for (uint32_t dst_by = 0; dst_by < dst_num_blocks_y; dst_by++) + { + uint8_t* pDst_block_u8 = (uint8_t*)pDst_blocks + dst_by * output_row_pitch_in_blocks_or_pixels * output_block_or_pixel_stride_in_bytes; + + for (uint32_t dst_bx = 0; dst_bx < dst_num_blocks_x; dst_bx++) + { + temp_image.extract_block_clamped(block_pixels, dst_bx * 4, dst_by * 4, 4, 4); + + transcode_4x4_block( + fmt, + dst_bx, dst_by, + pDst_blocks, pDst_block_u8, + block_pixels, + output_block_or_pixel_stride_in_bytes, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels, + channel0, channel1, + high_quality, from_alpha, + bc7f_flags, + etc1_pack_state, + dec_state.m_has_alpha ? 1 : 0); + + pDst_block_u8 += output_block_or_pixel_stride_in_bytes; + + } // dst_bx + + } // dst_by + + } // if (dst_fmt_is_pvrtc1) + } + else + { + // No PVRTC1/ASTC, no deblocking. Unpack as few source row blocks into memory as possible needed for transcoding to 4x4. Output block size must be 4x4. + assert((dst_fmt_block_width == 4) && (dst_fmt_block_height == 4)); + + // Compute how many source block rows we need to buffer so we have a multiple of 4 scanlines. The max # of scanlines is 20. + uint32_t num_src_block_rows_to_buffer = 1; + while ((num_src_block_rows_to_buffer * src_block_height) & 3) + num_src_block_rows_to_buffer++; + assert((num_src_block_rows_to_buffer >= 1) && (num_src_block_rows_to_buffer <= 4)); + + // Compute how many 4x4 dest blocks fit into these many source rows. + assert(((num_src_block_rows_to_buffer * src_block_height) & 3) == 0); + //const uint32_t num_dst_block_rows_to_buffer = (num_src_block_rows_to_buffer * src_block_height) >> 2; + + const uint32_t dst_num_blocks_x = (orig_width + dst_fmt_block_width - 1) / dst_fmt_block_width; + const uint32_t dst_num_blocks_y = (orig_height + dst_fmt_block_height - 1) / dst_fmt_block_height; + + basisu::vector2D buffered_rows(src_num_blocks_x * src_block_width, num_src_block_rows_to_buffer * src_block_height); + + struct decode_state + { + uint32_t m_orig_height; + + uint32_t m_src_num_blocks_x; + uint32_t m_src_num_blocks_y; + uint32_t m_src_block_width; + uint32_t m_src_block_height; + + uint32_t m_dst_num_blocks_x; + uint32_t m_dst_num_blocks_y; + + void* m_pDst_blocks; + uint32_t m_output_row_pitch_in_blocks_or_pixels; + uint32_t m_output_block_or_pixel_stride_in_bytes; + uint32_t m_output_rows_in_pixels; + + uint32_t m_num_src_block_rows_to_buffer; + //uint32_t m_num_dst_block_rows_to_buffer; + + basisu::vector2D* m_pBuffered_rows; + + bool m_used_srgb_astc_decode_mode; + bool m_has_alpha; + + block_format m_fmt; + int m_channel0, m_channel1; + bool m_high_quality; + bool m_from_alpha; + uint32_t m_bc7f_flags; + etc1f::pack_etc1_state* m_pEtc1_pack_state; + }; + + decode_state dec_state; + dec_state.m_orig_height = orig_height; + dec_state.m_src_num_blocks_x = src_num_blocks_x; + dec_state.m_src_num_blocks_y = src_num_blocks_y; + dec_state.m_src_block_width = src_block_width; + dec_state.m_src_block_height = src_block_height; + dec_state.m_dst_num_blocks_x = dst_num_blocks_x; + dec_state.m_dst_num_blocks_y = dst_num_blocks_y; + dec_state.m_pDst_blocks = pDst_blocks; + dec_state.m_output_row_pitch_in_blocks_or_pixels = output_row_pitch_in_blocks_or_pixels; + dec_state.m_output_block_or_pixel_stride_in_bytes = output_block_or_pixel_stride_in_bytes; + dec_state.m_output_rows_in_pixels = output_rows_in_pixels; + + dec_state.m_num_src_block_rows_to_buffer = num_src_block_rows_to_buffer; + //dec_state.m_num_dst_block_rows_to_buffer = num_dst_block_rows_to_buffer; + + dec_state.m_pBuffered_rows = &buffered_rows; + dec_state.m_used_srgb_astc_decode_mode = false; // will be set by init from the compressed stream's header + dec_state.m_has_alpha = true; // will be set by init from the compressed stream's header + + dec_state.m_fmt = fmt; + dec_state.m_channel0 = channel0; + dec_state.m_channel1 = channel1; + dec_state.m_high_quality = high_quality; + dec_state.m_from_alpha = from_alpha; + dec_state.m_bc7f_flags = bc7f_flags; + dec_state.m_pEtc1_pack_state = &etc1_pack_state; + + auto init_func = [](uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData) + { + BASISU_NOTE_UNUSED(srgb_decode_profile); + BASISU_NOTE_UNUSED(dct_q); + + if (basisu::g_debug_printf) + basisu::debug_printf("init_func: %u %u %u %u %u %f %u\n", num_blocks_x, num_blocks_y, block_width, block_height, srgb_decode_profile, dct_q, has_alpha); + + decode_state& state = *(decode_state*)pData; + if ((block_width != state.m_src_block_width) || (block_height != state.m_src_block_height)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (3)\n"); + return false; + } + if ((num_blocks_x != state.m_src_num_blocks_x) || (num_blocks_y != state.m_src_num_blocks_y)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: header validation failed (4)\n"); + return false; + } + + state.m_used_srgb_astc_decode_mode = srgb_decode_profile; + state.m_has_alpha = has_alpha; + + return true; + }; + + auto src_block_func = [](uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData) + { + decode_state& state = *(decode_state*)pData; + assert((bx < state.m_src_num_blocks_x) && (by < state.m_src_num_blocks_y)); + + // Unpack ASTC block, distribute to temp output buffer. + color32 block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + bool decode_status = astc_helpers::decode_block_xuastc_ldr(log_blk, block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8); + if (!decode_status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block_xuastc_ldr() failed\n"); + return false; + } + +#if defined(_DEBUG) || defined(DEBUG) + color32 alt_block_pixels[astc_helpers::MAX_BLOCK_PIXELS]; + if (!astc_helpers::decode_block(log_blk, alt_block_pixels, state.m_src_block_width, state.m_src_block_height, state.m_used_srgb_astc_decode_mode ? astc_helpers::cDecodeModeSRGB8 : astc_helpers::cDecodeModeLDR8)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_helpers::decode_block() failed\n"); + return false; + } + + for (uint32_t i = 0; i < state.m_src_block_width * state.m_src_block_height; i++) + { + assert(block_pixels[i][0] == alt_block_pixels[i][0]); + assert(block_pixels[i][1] == alt_block_pixels[i][1]); + assert(block_pixels[i][2] == alt_block_pixels[i][2]); + assert(block_pixels[i][3] == alt_block_pixels[i][3]); + } +#endif + + // TODO: For uncompressed outputs, we could write directly to the output buffer, skipping buffering. + const uint32_t buffered_src_block_row_y = (by % state.m_num_src_block_rows_to_buffer); + + color32* pSrc_pixels = block_pixels; + color32* pDst_pixels = &(*state.m_pBuffered_rows)(bx * state.m_src_block_width, buffered_src_block_row_y * state.m_src_block_height); + + for (uint32_t y = 0; y < state.m_src_block_height; y++) + { + memcpy(pDst_pixels, pSrc_pixels, state.m_src_block_width * sizeof(color32)); + + pSrc_pixels += state.m_src_block_width; + pDst_pixels += state.m_pBuffered_rows->get_width(); + } // y + + // Last block on this src row? If not, exit. + if (bx != (state.m_src_num_blocks_x - 1)) + return true; + + // We've written the final src block for this ASTC src row. + // See if we have enough source rows to create full 4x4 destination blocks. + const bool final_src_block_row = (by == (state.m_src_num_blocks_y - 1)); + + if ( (buffered_src_block_row_y != (state.m_num_src_block_rows_to_buffer - 1)) && (!final_src_block_row) ) + return true; + + // src/destination image Y coordinate of the top of the buffered rows + const uint32_t buffered_src_pixel_y = ((by / state.m_num_src_block_rows_to_buffer) * state.m_num_src_block_rows_to_buffer) * state.m_src_block_height; + assert((buffered_src_pixel_y & 3) == 0); + + // The total # of valid src block rows we can read. + const uint32_t num_buffered_src_block_rows = buffered_src_block_row_y + 1; + + assert((num_buffered_src_block_rows == state.m_num_src_block_rows_to_buffer) || (final_src_block_row)); + + // The maximum number of valid buffer scanlines we can fetch from, taking into account the original texture's actual (unpadded) height. + const uint32_t override_buffer_height = basisu::minimum(state.m_orig_height - buffered_src_pixel_y, num_buffered_src_block_rows * state.m_src_block_height); + assert(override_buffer_height); + + // total_dst_block_rows_to_emit=really an upper bound for the final row of src ASTC blocks + const uint32_t total_dst_block_rows_to_emit = (num_buffered_src_block_rows * state.m_src_block_height + 3) >> 2; + + for (uint32_t dst_ofs_by = 0; dst_ofs_by < total_dst_block_rows_to_emit; dst_ofs_by++) + { + const uint32_t dst_by = (buffered_src_pixel_y >> 2) + dst_ofs_by; + if (dst_by >= state.m_dst_num_blocks_y) + break; + + for (uint32_t dst_bx = 0; dst_bx < state.m_dst_num_blocks_x; dst_bx++) + { + // Extract the 4x4 block pixels from our buffered rows, taking into account the actual # of valid scanlines inside the buffer. + state.m_pBuffered_rows->extract_block_clamped(block_pixels, dst_bx * 4, dst_ofs_by * 4, 4, 4, override_buffer_height); + + uint8_t* pDst_block_u8 = (uint8_t*)state.m_pDst_blocks + (dst_by * state.m_output_row_pitch_in_blocks_or_pixels + dst_bx) * state.m_output_block_or_pixel_stride_in_bytes; + + transcode_4x4_block( + state.m_fmt, + dst_bx, dst_by, + state.m_pDst_blocks, pDst_block_u8, + block_pixels, + state.m_output_block_or_pixel_stride_in_bytes, state.m_output_row_pitch_in_blocks_or_pixels, state.m_output_rows_in_pixels, + state.m_channel0, state.m_channel1, + state.m_high_quality, state.m_from_alpha, + state.m_bc7f_flags, + *state.m_pEtc1_pack_state, + state.m_has_alpha ? 1 : 0); + + } // dst_bx + + } // dst_ofs_by + + return true; + }; + + xuastc_decoded_image decoded_image; + + const bool decomp_flag = decoded_image.decode(pImage_data, image_data_size, init_func, &dec_state, src_block_func, &dec_state); + if (!decomp_flag) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: astc_ldr_t::decompress_image() failed\n"); + return false; + } + } + + } // if (basis_tex_format_is_astc_ldr(src_format)) + + return true; +#else + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_slice: XUASTC support disabled\n"); + return false; +#endif // BASISD_SUPPORT_XUASTC +} + +// Container independent transcoding +bool basisu_lowlevel_xuastc_ldr_transcoder::transcode_image( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, + transcoder_texture_format target_format, + void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const uint8_t* pCompressed_data, uint32_t compressed_data_length, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint64_t slice_offset, uint32_t slice_length, + uint32_t decode_flags, + bool has_alpha, + bool is_video, + uint32_t output_row_pitch_in_blocks_or_pixels, + basisu_transcoder_state* pState, + uint32_t output_rows_in_pixels, + int channel0, int channel1) +{ + BASISU_NOTE_UNUSED(is_video); + BASISU_NOTE_UNUSED(level_index); + +#if BASISD_SUPPORT_XUASTC + + if (((uint64_t)slice_offset + slice_length) > (uint64_t)compressed_data_length) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: source data buffer too small\n"); + return false; + } + + if ((target_format == transcoder_texture_format::cTFPVRTC1_4_RGB) || (target_format == transcoder_texture_format::cTFPVRTC1_4_RGBA)) + { + if ((!basisu::is_pow2(orig_width)) || (!basisu::is_pow2(orig_height))) + { + // PVRTC1 only supports power of 2 dimensions + BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_image: PVRTC1 only supports power of 2 dimensions\n"); + return false; + } + } + + const bool transcode_alpha_data_to_opaque_formats = (decode_flags & cDecodeFlagsTranscodeAlphaDataToOpaqueFormats) != 0; + const uint32_t bytes_per_block_or_pixel = basis_get_bytes_per_block_or_pixel(target_format); + + if (!basis_validate_output_buffer_size(target_format, output_blocks_buf_size_in_blocks_or_pixels, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, output_rows_in_pixels)) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: output buffer size too small\n"); + return false; + } + + bool status = false; + + switch (target_format) + { + case transcoder_texture_format::cTFETC1_RGB: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC1, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ETC1 failed\n"); + } + break; + } + case transcoder_texture_format::cTFETC2_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_RGBA, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ETC2 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC1_RGB: + { + // TODO: ETC1S allows BC1 from alpha channel. That doesn't seem actually useful, though. + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC1, + bytes_per_block_or_pixel, true, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC1 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC3_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC3, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC3 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC4_R: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC4, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC4 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC5_RG: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC5, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + 0, 3, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC5 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBC7_RGBA: + case transcoder_texture_format::cTFBC7_ALT: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBC7, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to BC7 failed\n"); + } + break; + } + case transcoder_texture_format::cTFPVRTC1_4_RGB: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGB, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to PVRTC1_RGB failed\n"); + } + break; + } + case transcoder_texture_format::cTFPVRTC1_4_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cPVRTC1_4_RGBA, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, channel0, channel1, decode_flags); + + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to PVRTC1_RGBA failed\n"); + } + break; + } + case transcoder_texture_format::cTFASTC_LDR_4x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x4_RGBA: + case transcoder_texture_format::cTFASTC_LDR_5x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_6x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x5_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x6_RGBA: + case transcoder_texture_format::cTFASTC_LDR_8x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x8_RGBA: + case transcoder_texture_format::cTFASTC_LDR_10x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x10_RGBA: + case transcoder_texture_format::cTFASTC_LDR_12x12_RGBA: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, xuastc_get_block_format(target_format), + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to ASTC 4x4 failed\n"); + } + break; + } + case transcoder_texture_format::cTFATC_RGB: + case transcoder_texture_format::cTFATC_RGBA: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->ATC currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFFXT1_RGB: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->FXT1 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFPVRTC2_4_RGB: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFPVRTC2_4_RGBA: + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: UASTC LDR 4x4->PVRTC2 currently unsupported\n"); + return false; + } + case transcoder_texture_format::cTFETC2_EAC_R11: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_R11, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + ((has_alpha) && (transcode_alpha_data_to_opaque_formats)) ? 3 : 0, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to EAC R11 failed\n"); + } + break; + } + case transcoder_texture_format::cTFETC2_EAC_RG11: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cETC2_EAC_RG11, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, + 0, 3, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to EAC RG11 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGBA32: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA32, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGBA32 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGB565: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGB565, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGB565 failed\n"); + } + break; + } + case transcoder_texture_format::cTFBGR565: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cBGR565, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGB565 failed\n"); + } + break; + } + case transcoder_texture_format::cTFRGBA4444: + { + status = transcode_slice(src_format, use_astc_srgb_decode_profile, pOutput_blocks, src_num_blocks_x, src_num_blocks_y, pCompressed_data + slice_offset, slice_length, block_format::cRGBA4444, + bytes_per_block_or_pixel, false, has_alpha, orig_width, orig_height, output_row_pitch_in_blocks_or_pixels, pState, output_rows_in_pixels, -1, -1, decode_flags); + if (!status) + { + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: transcode_slice() to RGBA4444 failed\n"); + } + break; + } + default: + { + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: Invalid format\n"); + break; + } + } + + return status; +#else + assert(0); + BASISU_DEVEL_ERROR("basisu_lowlevel_xuastc_ldr_transcoder::transcode_image: XUASTC support disabled\n"); + return false; +#endif // BASISD_SUPPORT_XUASTC +} + } // namespace basist + diff --git a/transcoder/basisu_transcoder.h b/transcoder/basisu_transcoder.h index dc9329c..a2fc5b3 100644 --- a/transcoder/basisu_transcoder.h +++ b/transcoder/basisu_transcoder.h @@ -1,5 +1,5 @@ // basisu_transcoder.h -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// Also see basis_tex_format in basisu_file_headers.h (TODO: Perhaps move key definitions into here.) #pragma once // By default KTX2 support is enabled to simplify compilation. This implies the need for the Zstandard library (which we distribute as a single source file in the "zstd" directory) by default. @@ -22,17 +24,11 @@ #define BASISD_SUPPORT_KTX2 1 #endif -// Set BASISD_SUPPORT_KTX2_ZSTD to 0 to disable Zstd usage and KTX2 UASTC Zstd supercompression support +// Set BASISD_SUPPORT_KTX2_ZSTD to 0 to disable Zstd usage and KTX2 UASTC Zstd supercompression support #ifndef BASISD_SUPPORT_KTX2_ZSTD #define BASISD_SUPPORT_KTX2_ZSTD 1 #endif -// Set BASISU_FORCE_DEVEL_MESSAGES to 1 to enable debug printf()'s whenever an error occurs, for easier debugging during development. -#ifndef BASISU_FORCE_DEVEL_MESSAGES - // TODO - disable before checking in - #define BASISU_FORCE_DEVEL_MESSAGES 0 -#endif - #include "basisu_transcoder_internal.h" #include "basisu_transcoder_uastc.h" #include "basisu_file_headers.h" @@ -42,7 +38,7 @@ namespace basist // High-level composite texture formats supported by the transcoder. // Each of these texture formats directly correspond to OpenGL/D3D/Vulkan etc. texture formats. // Notes: - // - If you specify a texture format that supports alpha, but the .basis file doesn't have alpha, the transcoder will automatically output a + // - If you specify a texture format that supports alpha, but the .basis file doesn't have alpha, the transcoder will automatically output a // fully opaque (255) alpha channel. // - The PVRTC1 texture formats only support power of 2 dimension .basis files, but this may be relaxed in a future version. // - The PVRTC1 transcoders are real-time encoders, so don't expect the highest quality. We may add a slower encoder with improved quality. @@ -66,13 +62,13 @@ namespace basist cTFPVRTC1_4_RGB = 8, // Opaque only, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified, nearly lowest quality of any texture format. cTFPVRTC1_4_RGBA = 9, // Opaque+alpha, most useful for simple opacity maps. If .basis file doesn't have alpha cTFPVRTC1_4_RGB will be used instead. Lowest quality of any supported texture format. - // ASTC (mobile, Intel devices, hopefully all desktop GPU's one day) - cTFASTC_4x4_RGBA = 10, // LDR. Opaque+alpha, ASTC 4x4, alpha channel will be opaque for opaque .basis files. + // ASTC (mobile, some Intel CPU's, hopefully all desktop GPU's one day) + cTFASTC_LDR_4x4_RGBA = 10, // LDR. Opaque+alpha, ASTC 4x4, alpha channel will be opaque for opaque .basis files. // LDR: Transcoder uses RGB/RGBA/L/LA modes, void extent, and up to two ([0,47] and [0,255]) endpoint precisions. // ATC (mobile, Adreno devices, this is a niche format) cTFATC_RGB = 11, // Opaque, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified. ATI ATC (GL_ATC_RGB_AMD) - cTFATC_RGBA = 12, // Opaque+alpha, alpha channel will be opaque for opaque .basis files. ATI ATC (GL_ATC_RGBA_INTERPOLATED_ALPHA_AMD) + cTFATC_RGBA = 12, // Opaque+alpha, alpha channel will be opaque for opaque .basis files. ATI ATC (GL_ATC_RGBA_INTERPOLATED_ALPHA_AMD) // FXT1 (desktop, Intel devices, this is a super obscure format) cTFFXT1_RGB = 17, // Opaque only, uses exclusively CC_MIXED blocks. Notable for having a 8x4 block size. GL_3DFX_texture_compression_FXT1 is supported on Intel integrated GPU's (such as HD 630). @@ -94,7 +90,7 @@ namespace basist cTFRGB565 = 14, // 16bpp RGB image stored in raster (not block) order in memory, R at bit position 11 cTFBGR565 = 15, // 16bpp RGB image stored in raster (not block) order in memory, R at bit position 0 cTFRGBA4444 = 16, // 16bpp RGBA image stored in raster (not block) order in memory, R at bit position 12, A at bit position 0 - + // Note these uncompressed formats (HALF and 9E5) can only be transcoded to from HDR input files (UASTC HDR 4x4 or ASTC HDR 6x6). cTFRGB_HALF = 24, // 48bpp RGB half (16-bits/component, 3 components) cTFRGBA_HALF = 25, // 64bpp RGBA half (16-bits/component, 4 components) (A will always currently 1.0, UASTC_HDR doesn't support alpha) @@ -102,7 +98,23 @@ namespace basist cTFASTC_HDR_6x6_RGBA = 27, // HDR, RGBA (currently our ASTC HDR 6x6 encodes are only RGB), unsigned - cTFTotalTextureFormats = 28, + + // The remaining LDR ASTC block sizes, excluding 4x4 (which is above). There are 14 total valid ASTC LDR/HDR block sizes. + cTFASTC_LDR_5x4_RGBA = 28, + cTFASTC_LDR_5x5_RGBA = 29, + cTFASTC_LDR_6x5_RGBA = 30, + cTFASTC_LDR_6x6_RGBA = 31, + cTFASTC_LDR_8x5_RGBA = 32, + cTFASTC_LDR_8x6_RGBA = 33, + cTFASTC_LDR_10x5_RGBA = 34, + cTFASTC_LDR_10x6_RGBA = 35, + cTFASTC_LDR_8x8_RGBA = 36, + cTFASTC_LDR_10x8_RGBA = 37, + cTFASTC_LDR_10x10_RGBA = 38, + cTFASTC_LDR_12x10_RGBA = 39, + cTFASTC_LDR_12x12_RGBA = 40, + + cTFTotalTextureFormats = 41, // ----- The following are old/legacy enums for compatibility with code compiled against previous versions cTFETC1 = cTFETC1_RGB, @@ -112,25 +124,30 @@ namespace basist cTFBC4 = cTFBC4_R, cTFBC5 = cTFBC5_RG, - // Previously, the caller had some control over which BC7 mode the transcoder output. We've simplified this due to UASTC, which supports numerous modes. + // Previously, the caller had some control over which BC7 mode the transcoder output. We've simplified this due to UASTC LDR 4x4, which supports numerous modes. cTFBC7_M6_RGB = cTFBC7_RGBA, // Opaque only, RGB or alpha if cDecodeFlagsTranscodeAlphaDataToOpaqueFormats flag is specified. Highest quality of all the non-ETC1 formats. cTFBC7_M5_RGBA = cTFBC7_RGBA, // Opaque+alpha, alpha channel will be opaque for opaque .basis files cTFBC7_M6_OPAQUE_ONLY = cTFBC7_RGBA, cTFBC7_M5 = cTFBC7_RGBA, cTFBC7_ALT = 7, - cTFASTC_4x4 = cTFASTC_4x4_RGBA, + cTFASTC_4x4 = cTFASTC_LDR_4x4_RGBA, cTFATC_RGBA_INTERPOLATED_ALPHA = cTFATC_RGBA, + + cTFASTC_4x4_RGBA = cTFASTC_LDR_4x4_RGBA }; // For compressed texture formats, this returns the # of bytes per block. For uncompressed, it returns the # of bytes per pixel. // NOTE: Previously, this function was called basis_get_bytes_per_block(), and it always returned 16*bytes_per_pixel for uncompressed formats which was confusing. uint32_t basis_get_bytes_per_block_or_pixel(transcoder_texture_format fmt); - // Returns format's name in ASCII + // Returns the transcoder texture format's name in ASCII const char* basis_get_format_name(transcoder_texture_format fmt); + // Returns basis texture format name in ASCII + const char* basis_get_tex_format_name(basis_tex_format fmt); + // Returns block format name in ASCII const char* basis_get_block_format_name(block_format fmt); @@ -143,6 +160,9 @@ namespace basist // Returns true if the format is LDR. inline bool basis_transcoder_format_is_ldr(transcoder_texture_format fmt) { return !basis_transcoder_format_is_hdr(fmt); } + // Returns true if the format is an LDR or HDR ASTC format. + bool basis_is_transcoder_texture_format_astc(transcoder_texture_format fmt); + // Returns the basisu::texture_format corresponding to the specified transcoder_texture_format. basisu::texture_format basis_get_basisu_texture_format(transcoder_texture_format fmt); @@ -156,23 +176,32 @@ namespace basist uint32_t basis_get_uncompressed_bytes_per_pixel(transcoder_texture_format fmt); // Returns the block width for the specified texture format, which is currently either 4 or 8 for FXT1. - uint32_t basis_get_block_width(transcoder_texture_format tex_type); + uint32_t basis_get_block_width(transcoder_texture_format fmt); // Returns the block height for the specified texture format, which is currently always 4. - uint32_t basis_get_block_height(transcoder_texture_format tex_type); + uint32_t basis_get_block_height(transcoder_texture_format fmt); + + // ASTC/XUASTC LDR formats only: Given a basis_tex_format (mode or codec), return the corresponding ASTC basisu::texture_format with the proper block size from 4x4-12x12. + basisu::texture_format basis_get_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt); + + // For any given basis_tex_format (mode or codec), return the LDR/HDR ASTC transcoder texture format with the proper block size. + transcoder_texture_format basis_get_transcoder_texture_format_from_basis_tex_format(basis_tex_format fmt); + // basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format: same as basis_get_transcoder_texture_format_from_basis_tex_format (TODO: remove) + transcoder_texture_format basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(basis_tex_format fmt); - // Returns true if the specified format was enabled at compile time, and is supported for the specific basis/ktx2 texture format (ETC1S, UASTC, or UASTC HDR). + // Returns true if the specified format was enabled at compile time, and is supported for the specific basis/ktx2 texture format (ETC1S, UASTC, or UASTC HDR, or XUASTC LDR 4x4-12x12). + // For XUASTC the ASTC block size must match the transcoder_texture_format's ASTC block size. bool basis_is_format_supported(transcoder_texture_format tex_type, basis_tex_format fmt = basis_tex_format::cETC1S); // Returns the block width/height for the specified basis texture file format. uint32_t basis_tex_format_get_block_width(basis_tex_format fmt); uint32_t basis_tex_format_get_block_height(basis_tex_format fmt); - + bool basis_tex_format_is_hdr(basis_tex_format fmt); inline bool basis_tex_format_is_ldr(basis_tex_format fmt) { return !basis_tex_format_is_hdr(fmt); } - + // Validates that the output buffer is large enough to hold the entire transcoded texture. - // For uncompressed texture formats, most input parameters are in pixels, not blocks. Blocks are 4x4 pixels. + // For uncompressed texture formats, most input parameters are in pixels, not blocks. bool basis_validate_output_buffer_size(transcoder_texture_format target_format, uint32_t output_blocks_buf_size_in_blocks_or_pixels, uint32_t orig_width, uint32_t orig_height, @@ -199,7 +228,7 @@ namespace basist basisu::vector m_block_endpoint_preds[2]; enum { cMaxPrevFrameLevels = 16 }; - basisu::vector m_prev_frame_indices[2][cMaxPrevFrameLevels]; // [alpha_flag][level_index] + basisu::vector m_prev_frame_indices[2][cMaxPrevFrameLevels]; // [alpha_flag][level_index] void clear() { @@ -214,7 +243,46 @@ namespace basist }; // Low-level helper classes that do the actual transcoding. + enum basisu_decode_flags + { + // PVRTC1: decode non-pow2 ETC1S texture level to the next larger power of 2 (not implemented yet, but we're going to support it). Ignored if the slice's dimensions are already a power of 2. + cDecodeFlagsPVRTCDecodeToNextPow2 = 2, + // When decoding to an opaque texture format, if the basis file has alpha, decode the alpha slice instead of the color slice to the output texture format. + // This is primarily to allow decoding of textures with alpha to multiple ETC1 textures (one for color, another for alpha). + cDecodeFlagsTranscodeAlphaDataToOpaqueFormats = 4, + + // Forbid usage of BC1 3 color blocks (we don't support BC1 punchthrough alpha yet). + // This flag is used internally when decoding to BC3. + cDecodeFlagsBC1ForbidThreeColorBlocks = 8, + + // The output buffer contains alpha endpoint/selector indices. + // Used internally when decoding formats like ASTC that require both color and alpha data to be available when transcoding to the output format. + cDecodeFlagsOutputHasAlphaIndices = 16, + + // Enable slower, but higher quality transcoding for some formats. + // For ASTC/XUASTC->BC7, this enables partially analytical encoding vs. fully analytical. + cDecodeFlagsHighQuality = 32, + + // Disable ETC1S->BC7 adaptive chroma filtering, for much faster transcoding to BC7. + cDecodeFlagsNoETC1SChromaFiltering = 64, + + // Disable deblock filtering for XUASTC LDR transcoding to non-ASTC formats. + // For ASTC 8x6 or smaller block sizes, deblocking is always disabled unless you force it on using cDecodeFlagsForceDeblockFiltering. + cDecodeFlagsNoDeblockFiltering = 128, + + // More aggressive deblock filtering (only used when it's enabled) + cDecodeFlagsStrongerDeblockFiltering = 256, + + // Always apply deblocking, even for smaller ASTC block sizes (4x4-8x6). + cDecodeFlagsForceDeblockFiltering = 512, + + // By default XUASTC LDR 4x4, 6x6 and 8x6 are directly transcoded to BC7 without always requiring a full ASTC block unpack and analytical BC7 encode. This is 1.4x up to 3x faster in WASM. + // This trade offs some quality. The largest transcoding speed gain is achieved when the source XUASTC data isn't dual plane and only uses 1 subset. Otherwise the actual perf. gain is variable. + // To disable this optimization for all XUASTC block sizes and always use the fallback encoder, specify cDecodeFlagXUASTCLDRDisableFastBC7Transcoding. + cDecodeFlagXUASTCLDRDisableFastBC7Transcoding = 1024 + }; + // ETC1S class basisu_lowlevel_etc1s_transcoder { @@ -279,42 +347,20 @@ namespace basist typedef basisu::vector selector_vec; const selector_vec& get_selectors() const { return m_local_selectors; } - + private: const basisu_lowlevel_etc1s_transcoder* m_pGlobal_codebook; endpoint_vec m_local_endpoints; selector_vec m_local_selectors; - + huffman_decoding_table m_endpoint_pred_model, m_delta_endpoint_model, m_selector_model, m_selector_history_buf_rle_model; uint32_t m_selector_history_buf_size; basisu_transcoder_state m_def_state; }; - - enum basisu_decode_flags - { - // PVRTC1: decode non-pow2 ETC1S texture level to the next larger power of 2 (not implemented yet, but we're going to support it). Ignored if the slice's dimensions are already a power of 2. - cDecodeFlagsPVRTCDecodeToNextPow2 = 2, - - // When decoding to an opaque texture format, if the basis file has alpha, decode the alpha slice instead of the color slice to the output texture format. - // This is primarily to allow decoding of textures with alpha to multiple ETC1 textures (one for color, another for alpha). - cDecodeFlagsTranscodeAlphaDataToOpaqueFormats = 4, - - // Forbid usage of BC1 3 color blocks (we don't support BC1 punchthrough alpha yet). - // This flag is used internally when decoding to BC3. - cDecodeFlagsBC1ForbidThreeColorBlocks = 8, - - // The output buffer contains alpha endpoint/selector indices. - // Used internally when decoding formats like ASTC that require both color and alpha data to be available when transcoding to the output format. - cDecodeFlagsOutputHasAlphaIndices = 16, - - cDecodeFlagsHighQuality = 32, - - cDecodeFlagsNoETC1SChromaFiltering = 64 - }; - + // UASTC LDR 4x4 class basisu_lowlevel_uastc_ldr_4x4_transcoder { @@ -352,6 +398,76 @@ namespace basist int channel0 = -1, int channel1 = -1); }; +#if BASISD_SUPPORT_XUASTC + // XUASTC LDR 4x4-12x12 or ASTC LDR 4x4-12x12 + struct xuastc_decoded_image + { + uint32_t m_actual_block_width = 0, m_actual_block_height = 0, m_actual_width = 0, m_actual_height = 0; + bool m_actual_has_alpha = false, m_uses_srgb_astc_decode_mode = false; + + bool decode(const uint8_t* pImage_data, uint32_t image_data_size, + astc_ldr_t::xuastc_decomp_image_init_callback_ptr pInit_callback, void* pInit_callback_data, + astc_ldr_t::xuastc_decomp_image_block_callback_ptr pBlock_callback, void* pBlock_callback_data) + { + const bool decomp_flag = astc_ldr_t::xuastc_ldr_decompress_image(pImage_data, image_data_size, + m_actual_block_width, m_actual_block_height, + m_actual_width, m_actual_height, + m_actual_has_alpha, m_uses_srgb_astc_decode_mode, basisu::g_debug_printf, + pInit_callback, pInit_callback_data, + pBlock_callback, pBlock_callback_data); + + return decomp_flag; + } + + void clear() + { + m_actual_block_width = 0; + m_actual_block_height = 0; + m_actual_width = 0; + m_actual_height = 0; + m_actual_has_alpha = false; + m_uses_srgb_astc_decode_mode = false; + } + }; +#endif + + class basisu_lowlevel_xuastc_ldr_transcoder + { + friend class basisu_transcoder; + + public: + basisu_lowlevel_xuastc_ldr_transcoder(); + + bool transcode_slice(basis_tex_format src_format, bool use_astc_srgb_decode_profile, void* pDst_blocks, uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0); + + bool transcode_slice(basis_tex_format src_format, bool use_astc_srgb_decode_profile, void* pDst_blocks, uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, + uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, const basis_file_header& header, const basis_slice_desc& slice_desc, uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, uint32_t decode_flags = 0) + { + return transcode_slice(src_format, use_astc_srgb_decode_profile, pDst_blocks, src_num_blocks_x, src_num_blocks_y, pImage_data, image_data_size, fmt, + output_block_or_pixel_stride_in_bytes, bc1_allow_threecolor_blocks, (header.m_flags & cBASISHeaderFlagHasAlphaSlices) != 0, slice_desc.m_orig_width, slice_desc.m_orig_height, output_row_pitch_in_blocks_or_pixels, + pState, output_rows_in_pixels, channel0, channel1, decode_flags); + } + + // Container independent transcoding + bool transcode_image( + basis_tex_format src_format, bool use_astc_srgb_decode_profile, + transcoder_texture_format target_format, + void* pOutput_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const uint8_t* pCompressed_data, uint32_t compressed_data_length, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint64_t slice_offset, uint32_t slice_length, + uint32_t decode_flags = 0, + bool has_alpha = false, + bool is_video = false, + uint32_t output_row_pitch_in_blocks_or_pixels = 0, + basisu_transcoder_state* pState = nullptr, + uint32_t output_rows_in_pixels = 0, + int channel0 = -1, int channel1 = -1); + }; + // UASTC HDR 4x4 class basisu_lowlevel_uastc_hdr_4x4_transcoder { @@ -426,13 +542,13 @@ namespace basist int channel0 = -1, int channel1 = -1); }; - // ASTC HDR 6x6 intermediate - class basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder + // UASTC HDR 6x6 intermediate + class basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder { friend class basisu_transcoder; public: - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder(); + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder(); bool transcode_slice(void* pDst_blocks, uint32_t num_blocks_x, uint32_t num_blocks_y, const uint8_t* pImage_data, uint32_t image_data_size, block_format fmt, uint32_t output_block_or_pixel_stride_in_bytes, bool bc1_allow_threecolor_blocks, bool has_alpha, const uint32_t orig_width, const uint32_t orig_height, uint32_t output_row_pitch_in_blocks_or_pixels = 0, @@ -465,9 +581,11 @@ namespace basist struct basisu_slice_info { + // The image's ACTUAL dimensions in texels. uint32_t m_orig_width; uint32_t m_orig_height; + // The texture's dimensions in texels - always a multiple of the texture's underlying block size (4x4-12x12). uint32_t m_width; uint32_t m_height; @@ -497,9 +615,11 @@ namespace basist uint32_t m_image_index; uint32_t m_total_levels; + // The image's ACTUAL dimensions in texels. uint32_t m_orig_width; uint32_t m_orig_height; - + + // The texture's dimensions in texels - always a multiple of the texture's underlying block size (4x4-12x12). uint32_t m_width; uint32_t m_height; @@ -583,12 +703,13 @@ namespace basist uint32_t m_block_height; bool m_y_flipped; // true if the image was Y flipped + bool m_srgb; // true if the image is sRGB, false if linear bool m_etc1s; // true if the file is ETC1S bool m_has_alpha_slices; // true if the texture has alpha slices (for ETC1S: even slices RGB, odd slices alpha) }; // High-level transcoder class which accepts .basis file data and allows the caller to query information about the file and transcode image levels to various texture formats. - // If you're just starting out this is the class you care about. + // If you're just starting out this is the class you care about (or see the KTX2 transcoder below). class basisu_transcoder { basisu_transcoder(basisu_transcoder&); @@ -639,11 +760,11 @@ namespace basist // transcode_image_level() decodes a single mipmap level from the .basis file to any of the supported output texture formats. // It'll first find the slice(s) to transcode, then call transcode_slice() one or two times to decode both the color and alpha texture data (or RG texture data from two slices for BC5). // If the .basis file doesn't have alpha slices, the output alpha blocks will be set to fully opaque (all 255's). - // Currently, to decode to PVRTC1 the basis texture's dimensions in pixels must be a power of 2, due to PVRTC1 format requirements. + // Currently, to decode to PVRTC1 the basis texture's dimensions in pixels must be a power of 2, due to PVRTC1 format requirements. // output_blocks_buf_size_in_blocks_or_pixels should be at least the image level's total_blocks (num_blocks_x * num_blocks_y), or the total number of output pixels if fmt==cTFRGBA32 etc. // output_row_pitch_in_blocks_or_pixels: Number of blocks or pixels per row. If 0, the transcoder uses the slice's num_blocks_x or orig_width (NOT num_blocks_x * 4). Ignored for PVRTC1 (due to texture swizzling). // output_rows_in_pixels: Ignored unless fmt is uncompressed (cRGBA32, etc.). The total number of output rows in the output buffer. If 0, the transcoder assumes the slice's orig_height (NOT num_blocks_y * 4). - // Notes: + // Notes: // - basisu_transcoder_init() must have been called first to initialize the transcoder lookup tables before calling this function. // - This method assumes the output texture buffer is readable. In some cases to handle alpha, the transcoder will write temporary data to the output texture in // a first pass, which will be read in a second pass. @@ -682,15 +803,16 @@ namespace basist const basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() const { return m_lowlevel_etc1s_decoder; } basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() { return m_lowlevel_etc1s_decoder; } - const basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() const { return m_lowlevel_uastc_decoder; } - basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() { return m_lowlevel_uastc_decoder; } + const basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() const { return m_lowlevel_uastc_ldr_4x4_decoder; } + basisu_lowlevel_uastc_ldr_4x4_transcoder& get_lowlevel_uastc_decoder() { return m_lowlevel_uastc_ldr_4x4_decoder; } private: mutable basisu_lowlevel_etc1s_transcoder m_lowlevel_etc1s_decoder; - mutable basisu_lowlevel_uastc_ldr_4x4_transcoder m_lowlevel_uastc_decoder; + mutable basisu_lowlevel_uastc_ldr_4x4_transcoder m_lowlevel_uastc_ldr_4x4_decoder; + mutable basisu_lowlevel_xuastc_ldr_transcoder m_lowlevel_xuastc_ldr_decoder; mutable basisu_lowlevel_uastc_hdr_4x4_transcoder m_lowlevel_uastc_4x4_hdr_decoder; mutable basisu_lowlevel_astc_hdr_6x6_transcoder m_lowlevel_astc_6x6_hdr_decoder; - mutable basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder m_lowlevel_astc_6x6_hdr_intermediate_decoder; + mutable basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder m_lowlevel_astc_6x6_hdr_intermediate_decoder; bool m_ready_to_transcode; @@ -701,7 +823,7 @@ namespace basist // basisu_transcoder_init() MUST be called before a .basis file can be transcoded. void basisu_transcoder_init(); - + enum debug_flags_t { cDebugFlagVisCRs = 1, @@ -711,10 +833,10 @@ namespace basist uint32_t get_debug_flags(); void set_debug_flags(uint32_t f); - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ // Optional .KTX2 file format support // KTX2 reading optionally requires miniz or Zstd decompressors for supercompressed UASTC files. - // ------------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------------ #if BASISD_SUPPORT_KTX2 #pragma pack(push) #pragma pack(1) @@ -764,10 +886,10 @@ namespace basist basisu::packed_uint<4> m_alpha_slice_byte_length; }; - struct ktx2_astc_hdr_6x6_intermediate_image_desc + struct ktx2_slice_offset_len_desc { - basisu::packed_uint<4> m_rgb_slice_byte_offset; - basisu::packed_uint<4> m_rgb_slice_byte_length; + basisu::packed_uint<4> m_slice_byte_offset; + basisu::packed_uint<4> m_slice_byte_length; }; struct ktx2_animdata @@ -779,7 +901,7 @@ namespace basist #pragma pack(pop) const uint32_t KTX2_VK_FORMAT_UNDEFINED = 0; - + // These are standard Vulkan texture VkFormat ID's, see https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkFormat.html const uint32_t KTX2_FORMAT_ASTC_4x4_SFLOAT_BLOCK = 1000066000; const uint32_t KTX2_FORMAT_ASTC_5x4_SFLOAT_BLOCK = 1000066001; @@ -789,12 +911,28 @@ namespace basist const uint32_t KTX2_FORMAT_ASTC_8x5_SFLOAT_BLOCK = 1000066005; const uint32_t KTX2_FORMAT_ASTC_8x6_SFLOAT_BLOCK = 1000066006; + const uint32_t KTX2_FORMAT_ASTC_4x4_UNORM_BLOCK = 157, KTX2_FORMAT_ASTC_4x4_SRGB_BLOCK = 158; + const uint32_t KTX2_FORMAT_ASTC_5x4_UNORM_BLOCK = 159, KTX2_FORMAT_ASTC_5x4_SRGB_BLOCK = 160; + const uint32_t KTX2_FORMAT_ASTC_5x5_UNORM_BLOCK = 161, KTX2_FORMAT_ASTC_5x5_SRGB_BLOCK = 162; + const uint32_t KTX2_FORMAT_ASTC_6x5_UNORM_BLOCK = 163, KTX2_FORMAT_ASTC_6x5_SRGB_BLOCK = 164; + const uint32_t KTX2_FORMAT_ASTC_6x6_UNORM_BLOCK = 165, KTX2_FORMAT_ASTC_6x6_SRGB_BLOCK = 166; + const uint32_t KTX2_FORMAT_ASTC_8x5_UNORM_BLOCK = 167, KTX2_FORMAT_ASTC_8x5_SRGB_BLOCK = 168; + const uint32_t KTX2_FORMAT_ASTC_8x6_UNORM_BLOCK = 169, KTX2_FORMAT_ASTC_8x6_SRGB_BLOCK = 170; + const uint32_t KTX2_FORMAT_ASTC_10x5_UNORM_BLOCK = 173, KTX2_FORMAT_ASTC_10x5_SRGB_BLOCK = 174; + const uint32_t KTX2_FORMAT_ASTC_10x6_UNORM_BLOCK = 175, KTX2_FORMAT_ASTC_10x6_SRGB_BLOCK = 176; + const uint32_t KTX2_FORMAT_ASTC_8x8_UNORM_BLOCK = 171, KTX2_FORMAT_ASTC_8x8_SRGB_BLOCK = 172; // note the ASTC block size order is off in the vkFormat definitions + const uint32_t KTX2_FORMAT_ASTC_10x8_UNORM_BLOCK = 177, KTX2_FORMAT_ASTC_10x8_SRGB_BLOCK = 178; + const uint32_t KTX2_FORMAT_ASTC_10x10_UNORM_BLOCK = 179, KTX2_FORMAT_ASTC_10x10_SRGB_BLOCK = 180; + const uint32_t KTX2_FORMAT_ASTC_12x10_UNORM_BLOCK = 181, KTX2_FORMAT_ASTC_12x10_SRGB_BLOCK = 182; + const uint32_t KTX2_FORMAT_ASTC_12x12_UNORM_BLOCK = 183, KTX2_FORMAT_ASTC_12x12_SRGB_BLOCK = 184; + const uint32_t KTX2_KDF_DF_MODEL_ASTC = 162; // 0xA2 const uint32_t KTX2_KDF_DF_MODEL_ETC1S = 163; // 0xA3 const uint32_t KTX2_KDF_DF_MODEL_UASTC_LDR_4X4 = 166; // 0xA6 const uint32_t KTX2_KDF_DF_MODEL_UASTC_HDR_4X4 = 167; // 0xA7 - const uint32_t KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE = 168; // 0xA8, TODO - coordinate with Khronos on this - + const uint32_t KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE = 168; // 0xA8, TODO - coordinate with Khronos on this + const uint32_t KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE = 169; // 0xA9, TODO - coordinate with Khronos on this + const uint32_t KTX2_IMAGE_IS_P_FRAME = 2; const uint32_t KTX2_UASTC_BLOCK_SIZE = 16; // also the block size for UASTC_HDR const uint32_t KTX2_MAX_SUPPORTED_LEVEL_COUNT = 16; // this is an implementation specific constraint and can be increased @@ -878,12 +1016,12 @@ namespace basist { case KTX2_DF_PRIMARIES_UNSPECIFIED: return "UNSPECIFIED"; case KTX2_DF_PRIMARIES_BT709: return "BT709"; - case KTX2_DF_PRIMARIES_BT601_EBU: return "EBU"; + case KTX2_DF_PRIMARIES_BT601_EBU: return "EBU"; case KTX2_DF_PRIMARIES_BT601_SMPTE: return "SMPTE"; case KTX2_DF_PRIMARIES_BT2020: return "BT2020"; case KTX2_DF_PRIMARIES_CIEXYZ: return "CIEXYZ"; case KTX2_DF_PRIMARIES_ACES: return "ACES"; - case KTX2_DF_PRIMARIES_ACESCC: return "ACESCC"; + case KTX2_DF_PRIMARIES_ACESCC: return "ACESCC"; case KTX2_DF_PRIMARIES_NTSC1953: return "NTSC1953"; case KTX2_DF_PRIMARIES_PAL525: return "PAL525"; case KTX2_DF_PRIMARIES_DISPLAYP3: return "DISPLAYP3"; @@ -891,7 +1029,7 @@ namespace basist default: break; } return "?"; - } + } // Information about a single 2D texture "image" in a KTX2 file. struct ktx2_image_level_info @@ -901,19 +1039,19 @@ namespace basist uint32_t m_layer_index; uint32_t m_face_index; - // The image's actual (or the original source image's) width/height in pixels, which may not be divisible by 4 pixels. + // The image's ACTUAL (or the original source image's) width/height in pixels, which may not be divisible by the block size (4-12 pixels). uint32_t m_orig_width; uint32_t m_orig_height; - // The image's physical width/height, which will always be divisible by 4 pixels. + // The image's physical width/height, which will always be divisible by the format's block size (4-12 pixels). uint32_t m_width; uint32_t m_height; - - // The texture's dimensions in 4x4 or 6x6 texel blocks. + + // The texture's dimensions in 4x4-12x12 texel blocks. uint32_t m_num_blocks_x; uint32_t m_num_blocks_y; - // The format's block width/height (currently either 4 or 6). + // The format's block width/height (4-12). uint32_t m_block_width; uint32_t m_block_height; @@ -926,7 +1064,7 @@ namespace basist // true if the image is an I-Frame. Currently, for ETC1S textures, the first frame will always be an I-Frame, and subsequent frames will always be P-Frames. bool m_iframe_flag; }; - + // Thread-specific ETC1S/supercompressed UASTC transcoder state. (If you're not doing multithreading transcoding you can ignore this.) struct ktx2_transcoder_state { @@ -944,9 +1082,9 @@ namespace basist // This class is quite similar to basisu_transcoder. It treats KTX2 files as a simple container for ETC1S/UASTC texture data. // It does not support 1D or 3D textures. - // It only supports 2D and cubemap textures, with or without mipmaps, texture arrays of 2D/cubemap textures, and texture video files. + // It only supports 2D and cubemap textures, with or without mipmaps, texture arrays of 2D/cubemap textures, and texture video files. // It only supports raw non-supercompressed UASTC, ETC1S, UASTC+Zstd, or UASTC+zlib compressed files. - // DFD (Data Format Descriptor) parsing is purposely as simple as possible. + // DFD (Data Format Descriptor) parsing is purposely as simple as possible. // If you need to know how to interpret the texture channels you'll need to parse the DFD yourself after calling get_dfd(). class ktx2_transcoder { @@ -971,10 +1109,10 @@ namespace basist // Returns the KTX2 level index array. There will be one entry for each mipmap level. Valid after init(). const basisu::vector& get_level_index() const { return m_levels; } - // Returns the texture's width in texels. Always non-zero, might not be divisible by 4. Valid after init(). + // Returns the texture's width in texels. Always non-zero, might not be divisible by the block size. Valid after init(). uint32_t get_width() const { return m_header.m_pixel_width; } - // Returns the texture's height in texels. Always non-zero, might not be divisible by 4. Valid after init(). + // Returns the texture's height in texels. Always non-zero, might not be divisible by the block size. Valid after init(). uint32_t get_height() const { return m_header.m_pixel_height; } // Returns the texture's number of mipmap levels. Always returns 1 or higher. Valid after init(). @@ -986,15 +1124,15 @@ namespace basist // Returns 0 or the number of layers in the texture array or texture video. Valid after init(). uint32_t get_layers() const { return m_header.m_layer_count; } - // Returns cETC1S, cUASTC4x4, cUASTC_HDR_4x4, cASTC_HDR_6x6, cASTC_HDR_6x6_INTERMEDIATE. Valid after init(). + // Returns cETC1S, cUASTC4x4, cUASTC_HDR_4x4, cASTC_HDR_6x6, cUASTC_HDR_6x6_INTERMEDIATE, etc. Valid after init(). basist::basis_tex_format get_basis_tex_format() const { return m_format; } // ETC1S LDR 4x4 bool is_etc1s() const { return get_basis_tex_format() == basist::basis_tex_format::cETC1S; } // UASTC LDR 4x4 (only) - bool is_uastc() const { return get_basis_tex_format() == basist::basis_tex_format::cUASTC4x4; } - + bool is_uastc() const { return get_basis_tex_format() == basist::basis_tex_format::cUASTC_LDR_4x4; } + // Is ASTC HDR 4x4 or 6x6 bool is_hdr() const { @@ -1006,18 +1144,26 @@ namespace basist return !is_hdr(); } + // is UASTC HDR 4x4 (which is also standard ASTC HDR 4x4 data) bool is_hdr_4x4() const { return (get_basis_tex_format() == basist::basis_tex_format::cUASTC_HDR_4x4); } + // is ASTC HDR 6x6 or UASTC HDR 6x6 intermediate (only) bool is_hdr_6x6() const { - return (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6) || (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + return (get_basis_tex_format() == basist::basis_tex_format::cASTC_HDR_6x6) || (get_basis_tex_format() == basist::basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); } + + // is ASTC LDR 4x4-12x12 (only) + bool is_astc_ldr() const { return basis_tex_format_is_astc_ldr(get_basis_tex_format()); } + + // is XUASTC LDR 4x4-12x12 (only) + bool is_xuastc_ldr() const { return basis_tex_format_is_xuastc_ldr(get_basis_tex_format()); } uint32_t get_block_width() const { return basis_tex_format_get_block_width(get_basis_tex_format()); } - uint32_t get_block_height() const { return basis_tex_format_get_block_height(get_basis_tex_format()); } + uint32_t get_block_height() const { return basis_tex_format_get_block_height(get_basis_tex_format()); } // Returns true if the ETC1S file has two planes (typically RGBA, or RRRG), or true if the UASTC file has alpha data. Valid after init(). uint32_t get_has_alpha() const { return m_has_alpha; } @@ -1032,17 +1178,19 @@ namespace basist // Returns the DFD color primary. // We do not validate the color primaries, so the returned value may not be in the ktx2_df_color_primaries enum. ktx2_df_color_primaries get_dfd_color_primaries() const { return m_dfd_color_prims; } - + // Returns KTX2_KHR_DF_TRANSFER_LINEAR or KTX2_KHR_DF_TRANSFER_SRGB. uint32_t get_dfd_transfer_func() const { return m_dfd_transfer_func; } + bool is_srgb() const { return (get_dfd_transfer_func() == KTX2_KHR_DF_TRANSFER_SRGB); } + uint32_t get_dfd_flags() const { return m_dfd_flags; } // Returns 1 (ETC1S/UASTC) or 2 (ETC1S with an internal alpha channel). uint32_t get_dfd_total_samples() const { return m_dfd_samples; } - - // Returns the channel mapping for each DFD "sample". UASTC always has 1 sample, ETC1S can have one or two. - // Note the returned value SHOULD be one of the ktx2_df_channel_id enums, but we don't validate that. + + // Returns the channel mapping for each DFD "sample". UASTC always has 1 sample, ETC1S can have one or two. + // Note the returned value SHOULD be one of the ktx2_df_channel_id enums, but we don't validate that. // It's up to the caller to decide what to do if the value isn't in the enum. ktx2_df_channel_id get_dfd_channel_id0() const { return m_dfd_chan0; } ktx2_df_channel_id get_dfd_channel_id1() const { return m_dfd_chan1; } @@ -1050,11 +1198,11 @@ namespace basist // Key value field data. struct key_value { - // The key field is UTF8 and always zero terminated. + // The key field is UTF8 and always zero terminated. // In memory we always append a zero terminator to the key. basisu::uint8_vec m_key; - // The value may be empty. In the KTX2 file it consists of raw bytes which may or may not be zero terminated. + // The value may be empty. In the KTX2 file it consists of raw bytes which may or may not be zero terminated. // In memory we always append a zero terminator to the value. basisu::uint8_vec m_value; @@ -1076,7 +1224,7 @@ namespace basist // Returns the array of ETC1S image descriptors, which is only valid after get_etc1s_image_descs() is called. const basisu::vector& get_etc1s_image_descs() const { return m_etc1s_image_descs; } - const basisu::vector& get_astc_hdr_6x6_intermediate_image_descs() const { return m_astc_6x6_intermediate_image_descs; } + const basisu::vector& get_slice_offset_len_descs() const { return m_slice_offset_len_descs; } // Must have called startTranscoding() first uint32_t get_etc1s_image_descs_image_flags(uint32_t level_index, uint32_t layer_index, uint32_t face_index) const; @@ -1084,21 +1232,21 @@ namespace basist // is_video() is only valid after start_transcoding() is called. // For ETC1S data, if this returns true you must currently transcode the file from first to last frame, in order, without skipping any frames. bool is_video() const { return m_is_video; } - + // Defaults to 0, only non-zero if the key existed in the source KTX2 file. float get_ldr_hdr_upconversion_nit_multiplier() const { return m_ldr_hdr_upconversion_nit_multiplier; } - - // start_transcoding() MUST be called before calling transcode_image(). + + // start_transcoding() MUST be called before calling transcode_image_level(). // This method decompresses the ETC1S global endpoint/selector codebooks, which is not free, so try to avoid calling it excessively. bool start_transcoding(); - + // get_image_level_info() be called after init(), but the m_iframe_flag's won't be valid until start_transcoding() is called. // You can call this method before calling transcode_image_level() to retrieve basic information about the mipmap level's dimensions, etc. bool get_image_level_info(ktx2_image_level_info& level_info, uint32_t level_index, uint32_t layer_index, uint32_t face_index) const; // transcode_image_level() transcodes a single 2D texture or cubemap face from the KTX2 file. // Internally it uses the same low-level transcode API's as basisu_transcoder::transcode_image_level(). - // If the file is UASTC and is supercompressed with Zstandard, and the file is a texture array or cubemap, it's highly recommended that each mipmap level is + // If the file is UASTC and is supercompressed with Zstandard, and the file is a texture array or cubemap, it's highly recommended that each mipmap level is // completely transcoded before switching to another level. Every time the mipmap level is changed all supercompressed level data must be decompressed using Zstandard as a single unit. // Currently ETC1S videos must always be transcoded from first to last frame (or KTX2 "layer"), in order, with no skipping of frames. // By default this method is not thread safe unless you specify a pointer to a user allocated thread-specific transcoder_state struct. @@ -1108,7 +1256,7 @@ namespace basist basist::transcoder_texture_format fmt, uint32_t decode_flags = 0, uint32_t output_row_pitch_in_blocks_or_pixels = 0, uint32_t output_rows_in_pixels = 0, int channel0 = -1, int channel1 = -1, ktx2_transcoder_state *pState = nullptr); - + private: const uint8_t* m_pData; uint32_t m_data_size; @@ -1117,26 +1265,30 @@ namespace basist basisu::vector m_levels; basisu::uint8_vec m_dfd; key_value_vec m_key_values; - + ktx2_etc1s_global_data_header m_etc1s_header; basisu::vector m_etc1s_image_descs; - basisu::vector m_astc_6x6_intermediate_image_descs; + basisu::vector m_slice_offset_len_descs; basist::basis_tex_format m_format; - + uint32_t m_dfd_color_model; ktx2_df_color_primaries m_dfd_color_prims; - uint32_t m_dfd_transfer_func; + + // KTX2_KHR_DF_TRANSFER_LINEAR vs. KTX2_KHR_DF_TRANSFER_SRGB (for XUASTC LDR: which profile was used during encoding) + uint32_t m_dfd_transfer_func; + uint32_t m_dfd_flags; uint32_t m_dfd_samples; ktx2_df_channel_id m_dfd_chan0, m_dfd_chan1; - + basist::basisu_lowlevel_etc1s_transcoder m_etc1s_transcoder; - basist::basisu_lowlevel_uastc_ldr_4x4_transcoder m_uastc_transcoder; + basist::basisu_lowlevel_uastc_ldr_4x4_transcoder m_uastc_ldr_transcoder; + basist::basisu_lowlevel_xuastc_ldr_transcoder m_xuastc_ldr_transcoder; basist::basisu_lowlevel_uastc_hdr_4x4_transcoder m_uastc_hdr_transcoder; basist::basisu_lowlevel_astc_hdr_6x6_transcoder m_astc_hdr_6x6_transcoder; - basist::basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder m_astc_hdr_6x6_intermediate_transcoder; - + basist::basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder m_astc_hdr_6x6_intermediate_transcoder; + ktx2_transcoder_state m_def_transcoder_state; bool m_has_alpha; @@ -1144,7 +1296,7 @@ namespace basist float m_ldr_hdr_upconversion_nit_multiplier; bool decompress_level_data(uint32_t level_index, basisu::uint8_vec& uncomp_data); - bool read_astc_6x6_hdr_intermediate_global_data(); + bool read_slice_offset_len_global_data(); bool decompress_etc1s_global_data(); bool read_key_values(); }; @@ -1165,7 +1317,7 @@ namespace basist break; } } - + if (!p) p = key_values.enlarge(1); @@ -1189,3 +1341,4 @@ namespace basist bool basisu_transcoder_supports_ktx2_zstd(); } // namespace basisu + diff --git a/transcoder/basisu_transcoder_internal.h b/transcoder/basisu_transcoder_internal.h index 5480ba3..c830795 100644 --- a/transcoder/basisu_transcoder_internal.h +++ b/transcoder/basisu_transcoder_internal.h @@ -1,5 +1,5 @@ // basisu_transcoder_internal.h - Universal texture format transcoder library. -// Copyright (C) 2019-2024 Binomial LLC. All Rights Reserved. +// Copyright (C) 2019-2026 Binomial LLC. All Rights Reserved. // // Important: If compiling with gcc, be sure strict aliasing is disabled: -fno-strict-aliasing // @@ -22,8 +22,10 @@ // v1.50: Added UASTC HDR 4x4 support // v1.60: Added RDO ASTC HDR 6x6 and intermediate support -#define BASISD_LIB_VERSION 160 -#define BASISD_VERSION_STRING "01.60" +// v1.65: Added ASTC LDR 4x4-12x12 and XUASTC LDR 4x4-12x12 +// v2.00: Added unified effort/quality options across all formats, fast direct transcoding of XUASTC 4x4/6x6/8x6 to BC7, adaptive deblocking, ZStd or arithmetic profiles, weight grid DCT +#define BASISD_LIB_VERSION 200 +#define BASISD_VERSION_STRING "02.00" #ifdef _DEBUG #define BASISD_BUILD_DEBUG @@ -32,6 +34,7 @@ #endif #include "basisu.h" +#include "basisu_astc_helpers.h" #define BASISD_znew (z = 36969 * (z & 65535) + (z >> 16)) @@ -46,9 +49,9 @@ namespace basist // You probably don't care about these enum's unless you are going pretty low-level and calling the transcoder to decode individual slices. enum class block_format { - cETC1, // ETC1S RGB + cETC1, // ETC1S RGB cETC2_RGBA, // full ETC2 EAC RGBA8 block - cBC1, // DXT1 RGB + cBC1, // DXT1 RGB cBC3, // BC4 block followed by a four color BC1 block cBC4, // DXT5A (alpha block only) cBC5, // two BC4 blocks @@ -58,9 +61,9 @@ namespace basist cBC7_M5_COLOR, // RGB BC7 mode 5 color (writes an opaque mode 5 block) cBC7_M5_ALPHA, // alpha portion of BC7 mode 5 (cBC7_M5_COLOR output data must have been written to the output buffer first to set the mode/rot fields etc.) cETC2_EAC_A8, // alpha block of ETC2 EAC (first 8 bytes of the 16-bit ETC2 EAC RGBA format) - cASTC_4x4, // ASTC 4x4 (either color-only or color+alpha). Note that the transcoder always currently assumes sRGB is not enabled when outputting ASTC + cASTC_LDR_4x4, // ASTC LDR 4x4 (either color-only or color+alpha). Note that the transcoder always currently assumes sRGB decode mode is not enabled when outputting ASTC LDR for ETC1S/UASTC LDR 4x4. // data. If you use a sRGB ASTC format you'll get ~1 LSB of additional error, because of the different way ASTC decoders scale 8-bit endpoints to 16-bits during unpacking. - + cATC_RGB, cATC_RGBA_INTERPOLATED_ALPHA, cFXT1_RGB, // Opaque-only, has oddball 8x4 pixel block size @@ -70,16 +73,16 @@ namespace basist cETC2_EAC_R11, cETC2_EAC_RG11, - + cIndices, // Used internally: Write 16-bit endpoint and selector indices directly to output (output block must be at least 32-bits) cRGB32, // Writes RGB components to 32bpp output pixels cRGBA32, // Writes RGB255 components to 32bpp output pixels cA32, // Writes alpha component to 32bpp output pixels - + cRGB565, cBGR565, - + cRGBA4444_COLOR, cRGBA4444_ALPHA, cRGBA4444_COLOR_OPAQUE, @@ -91,12 +94,73 @@ namespace basist cUASTC_4x4, // LDR, universal cUASTC_HDR_4x4, // HDR, transcodes only to 4x4 HDR ASTC, BC6H, or uncompressed cBC6H, + cASTC_HDR_4x4, cASTC_HDR_6x6, + // The remaining ASTC LDR block sizes. + cASTC_LDR_5x4, + cASTC_LDR_5x5, + cASTC_LDR_6x5, + cASTC_LDR_6x6, + cASTC_LDR_8x5, + cASTC_LDR_8x6, + cASTC_LDR_10x5, + cASTC_LDR_10x6, + cASTC_LDR_8x8, + cASTC_LDR_10x8, + cASTC_LDR_10x10, + cASTC_LDR_12x10, + cASTC_LDR_12x12, + cTotalBlockFormats }; + inline bool block_format_is_hdr(block_format fmt) + { + switch (fmt) + { + case block_format::cUASTC_HDR_4x4: + case block_format::cBC6H: + case block_format::cASTC_HDR_4x4: + case block_format::cASTC_HDR_6x6: + return true; + default: + break; + } + + return false; + } + + // LDR or HDR ASTC? + inline bool block_format_is_astc(block_format fmt) + { + switch (fmt) + { + case block_format::cASTC_LDR_4x4: + case block_format::cASTC_LDR_5x4: + case block_format::cASTC_LDR_5x5: + case block_format::cASTC_LDR_6x5: + case block_format::cASTC_LDR_6x6: + case block_format::cASTC_LDR_8x5: + case block_format::cASTC_LDR_8x6: + case block_format::cASTC_LDR_10x5: + case block_format::cASTC_LDR_10x6: + case block_format::cASTC_LDR_8x8: + case block_format::cASTC_LDR_10x8: + case block_format::cASTC_LDR_10x10: + case block_format::cASTC_LDR_12x10: + case block_format::cASTC_LDR_12x12: + case block_format::cASTC_HDR_4x4: + case block_format::cASTC_HDR_6x6: + return true; + default: + break; + } + + return false; + } + inline uint32_t get_block_width(block_format fmt) { switch (fmt) @@ -105,6 +169,21 @@ namespace basist return 8; case block_format::cASTC_HDR_6x6: return 6; + + case block_format::cASTC_LDR_5x4: return 5; + case block_format::cASTC_LDR_5x5: return 5; + case block_format::cASTC_LDR_6x5: return 6; + case block_format::cASTC_LDR_6x6: return 6; + case block_format::cASTC_LDR_8x5: return 8; + case block_format::cASTC_LDR_8x6: return 8; + case block_format::cASTC_LDR_10x5: return 10; + case block_format::cASTC_LDR_10x6: return 10; + case block_format::cASTC_LDR_8x8: return 8; + case block_format::cASTC_LDR_10x8: return 10; + case block_format::cASTC_LDR_10x10: return 10; + case block_format::cASTC_LDR_12x10: return 12; + case block_format::cASTC_LDR_12x12: return 12; + default: break; } @@ -117,6 +196,20 @@ namespace basist { case block_format::cASTC_HDR_6x6: return 6; + + case block_format::cASTC_LDR_5x5: return 5; + case block_format::cASTC_LDR_6x5: return 5; + case block_format::cASTC_LDR_6x6: return 6; + case block_format::cASTC_LDR_8x5: return 5; + case block_format::cASTC_LDR_8x6: return 6; + case block_format::cASTC_LDR_10x5: return 5; + case block_format::cASTC_LDR_10x6: return 6; + case block_format::cASTC_LDR_8x8: return 8; + case block_format::cASTC_LDR_10x8: return 8; + case block_format::cASTC_LDR_10x10: return 10; + case block_format::cASTC_LDR_12x10: return 10; + case block_format::cASTC_LDR_12x12: return 12; + default: break; } @@ -140,9 +233,31 @@ namespace basist const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_THRESH = 3; const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_BITS = 6; const uint32_t SELECTOR_HISTORY_BUF_RLE_COUNT_TOTAL = (1 << SELECTOR_HISTORY_BUF_RLE_COUNT_BITS); - + uint16_t crc16(const void *r, size_t size, uint16_t crc); + uint32_t hash_hsieh(const uint8_t* pBuf, size_t len); + + template + struct bit_hasher + { + inline std::size_t operator()(const Key& k) const + { + return hash_hsieh(reinterpret_cast(&k), sizeof(k)); + } + }; + + struct string_hasher + { + inline std::size_t operator()(const std::string& k) const + { + size_t l = k.size(); + if (!l) + return 0; + return hash_hsieh(reinterpret_cast(k.c_str()), l); + } + }; + class huffman_decoding_table { friend class bitwise_decoder; @@ -260,7 +375,7 @@ namespace basist return false; else if (idx >= (int)m_tree.size()) m_tree.resize(idx + 1); - + if (!m_tree[idx]) { m_tree[idx] = (int16_t)tree_next; @@ -350,7 +465,7 @@ namespace basist void stop() { } - + inline uint32_t peek_bits(uint32_t num_bits) { if (!num_bits) @@ -429,14 +544,14 @@ namespace basist for (;;) { uint32_t k = peek_bits(16); - + uint32_t l = 0; while (k & 1) { l++; k >>= 1; } - + q += l; remove_bits(l); @@ -454,7 +569,7 @@ namespace basist const uint32_t chunk_size = 1 << chunk_bits; const uint32_t chunk_mask = chunk_size - 1; - + uint32_t v = 0; uint32_t ofs = 0; @@ -466,7 +581,7 @@ namespace basist if ((s & chunk_size) == 0) break; - + if (ofs >= 32) { assert(0); @@ -482,7 +597,7 @@ namespace basist assert(ct.m_code_sizes.size()); const uint32_t huffman_fast_lookup_size = 1 << fast_lookup_bits; - + while (m_bit_buf_size < 16) { uint32_t c = 0; @@ -493,7 +608,7 @@ namespace basist m_bit_buf_size += 8; assert(m_bit_buf_size <= 32); } - + int code_len; int sym; @@ -603,6 +718,100 @@ namespace basist uint32_t m_bit_buf_size; }; + class simplified_bitwise_decoder + { + public: + simplified_bitwise_decoder() : + m_pBuf(nullptr), + m_pBuf_end(nullptr), + m_bit_buf(0) + { + } + + void clear() + { + m_pBuf = nullptr; + m_pBuf_end = nullptr; + m_bit_buf = 0; + } + + bool init(const uint8_t* pBuf, size_t buf_size) + { + if ((!pBuf) && (buf_size)) + return false; + + m_pBuf = pBuf; + m_pBuf_end = pBuf + buf_size; + m_bit_buf = 1; + return true; + } + + bool init(const basisu::uint8_vec& buf) + { + return init(buf.data(), buf.size()); + } + + // num_bits must be 1, 2, 4 or 8 and codes cannot cross bytes + inline uint32_t get_bits(uint32_t num_bits) + { + assert(m_pBuf); + + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + + const uint32_t mask = (1 << num_bits) - 1; + const uint32_t res = m_bit_buf & mask; + m_bit_buf >>= num_bits; + assert(m_bit_buf >= 1); + + return res; + } + + inline uint32_t get_bits1() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 1; + m_bit_buf >>= 1; + assert(m_bit_buf >= 1); + return res; + } + + inline uint32_t get_bits2() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 3; + m_bit_buf >>= 2; + assert(m_bit_buf >= 1); + return res; + } + + inline uint32_t get_bits4() + { + assert(m_pBuf); + if (m_bit_buf <= 1) + m_bit_buf = 256 | ((m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0); + const uint32_t res = m_bit_buf & 15; + m_bit_buf >>= 4; + assert(m_bit_buf >= 1); + return res; + } + + // No bitbuffer, can only ever retrieve bytes correctly. + inline uint32_t get_bits8() + { + assert(m_pBuf); + return (m_pBuf < m_pBuf_end) ? *m_pBuf++ : 0; + } + + const uint8_t* m_pBuf; + const uint8_t* m_pBuf_end; + uint32_t m_bit_buf; + }; + inline uint32_t basisd_rand(uint32_t seed) { if (!seed) @@ -684,7 +893,7 @@ namespace basist }; struct decoder_etc_block; - + inline uint8_t clamp255(int32_t i) { return (uint8_t)((i & 0xFFFFFF00U) ? (~(i >> 31)) : i); @@ -708,11 +917,12 @@ namespace basist }; uint8_t c[4]; - + uint32_t m; }; - color32() { } + //color32() { } + color32() = default; color32(uint32_t vr, uint32_t vg, uint32_t vb, uint32_t va) { set(vr, vg, vb, va); } color32(eNoClamp unused, uint32_t vr, uint32_t vg, uint32_t vb, uint32_t va) { (void)unused; set_noclamp_rgba(vr, vg, vb, va); } @@ -744,6 +954,172 @@ namespace basist bool operator!= (const endpoint& rhs) const { return !(*this == rhs); } }; + // This duplicates key functionality in the encoder library's color_rgba class. Porting and retesting code that uses it to color32 is impractical. + class color_rgba + { + public: + union + { + uint8_t m_comps[4]; + + struct + { + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; + }; + }; + + inline color_rgba() + { + static_assert(sizeof(*this) == 4, "sizeof(*this) != 4"); + static_assert(sizeof(*this) == sizeof(color32), "sizeof(*this) != sizeof(basist::color32)"); + } + + inline color_rgba(const color32& other) : + r(other.r), + g(other.g), + b(other.b), + a(other.a) + { + } + + color_rgba& operator= (const basist::color32& rhs) + { + r = rhs.r; + g = rhs.g; + b = rhs.b; + a = rhs.a; + return *this; + } + + inline color_rgba(int y) + { + set(y); + } + + inline color_rgba(int y, int na) + { + set(y, na); + } + + inline color_rgba(int sr, int sg, int sb, int sa) + { + set(sr, sg, sb, sa); + } + + inline color_rgba(eNoClamp, int sr, int sg, int sb, int sa) + { + set_noclamp_rgba((uint8_t)sr, (uint8_t)sg, (uint8_t)sb, (uint8_t)sa); + } + + inline color_rgba& set_noclamp_y(int y) + { + m_comps[0] = (uint8_t)y; + m_comps[1] = (uint8_t)y; + m_comps[2] = (uint8_t)y; + m_comps[3] = (uint8_t)255; + return *this; + } + + inline color_rgba& set_noclamp_rgba(int sr, int sg, int sb, int sa) + { + m_comps[0] = (uint8_t)sr; + m_comps[1] = (uint8_t)sg; + m_comps[2] = (uint8_t)sb; + m_comps[3] = (uint8_t)sa; + return *this; + } + + inline color_rgba& set(int y) + { + m_comps[0] = static_cast(basisu::clamp(y, 0, 255)); + m_comps[1] = m_comps[0]; + m_comps[2] = m_comps[0]; + m_comps[3] = 255; + return *this; + } + + inline color_rgba& set(int y, int na) + { + m_comps[0] = static_cast(basisu::clamp(y, 0, 255)); + m_comps[1] = m_comps[0]; + m_comps[2] = m_comps[0]; + m_comps[3] = static_cast(basisu::clamp(na, 0, 255)); + return *this; + } + + inline color_rgba& set(int sr, int sg, int sb, int sa) + { + m_comps[0] = static_cast(basisu::clamp(sr, 0, 255)); + m_comps[1] = static_cast(basisu::clamp(sg, 0, 255)); + m_comps[2] = static_cast(basisu::clamp(sb, 0, 255)); + m_comps[3] = static_cast(basisu::clamp(sa, 0, 255)); + return *this; + } + + inline color_rgba& set_rgb(int sr, int sg, int sb) + { + m_comps[0] = static_cast(basisu::clamp(sr, 0, 255)); + m_comps[1] = static_cast(basisu::clamp(sg, 0, 255)); + m_comps[2] = static_cast(basisu::clamp(sb, 0, 255)); + return *this; + } + + inline color_rgba& set_rgb(const color_rgba& other) + { + r = other.r; + g = other.g; + b = other.b; + return *this; + } + + inline const uint8_t& operator[] (uint32_t index) const { assert(index < 4); return m_comps[index]; } + inline uint8_t& operator[] (uint32_t index) { assert(index < 4); return m_comps[index]; } + + inline void clear() + { + m_comps[0] = 0; + m_comps[1] = 0; + m_comps[2] = 0; + m_comps[3] = 0; + } + + inline bool operator== (const color_rgba& rhs) const + { + if (m_comps[0] != rhs.m_comps[0]) return false; + if (m_comps[1] != rhs.m_comps[1]) return false; + if (m_comps[2] != rhs.m_comps[2]) return false; + if (m_comps[3] != rhs.m_comps[3]) return false; + return true; + } + + inline bool operator!= (const color_rgba& rhs) const + { + return !(*this == rhs); + } + + inline bool operator<(const color_rgba& rhs) const + { + for (int i = 0; i < 4; i++) + { + if (m_comps[i] < rhs.m_comps[i]) + return true; + else if (m_comps[i] != rhs.m_comps[i]) + return false; + } + return false; + } + + inline color32 get_color32() const + { + return color32(r, g, b, a); + } + + inline int get_709_luma() const { return (13938U * m_comps[0] + 46869U * m_comps[1] + 4729U * m_comps[2] + 32768U) >> 16U; } + }; + struct selector { // Plain selectors (2-bits per value) @@ -989,7 +1365,7 @@ namespace basist extern const uint8_t g_bc6h_weight4[16]; extern const int8_t g_bc6h_mode_lookup[32]; - + // Converts b16 to half float inline half_float bc6h_blog16_to_half(uint32_t comp) { @@ -1003,7 +1379,7 @@ namespace basist const uint32_t MAX_BC6H_HALF_FLOAT_AS_UINT = 0x7BFF; // Inverts bc6h_blog16_to_half(). - // Returns the nearest blog16 given a half value. + // Returns the nearest blog16 given a half value. inline uint32_t bc6h_half_to_blog16(half_float h) { assert(h <= MAX_BC6H_HALF_FLOAT_AS_UINT); @@ -1044,10 +1420,1844 @@ namespace basist }; void pack_bc6h_block(bc6h_block& dst_blk, bc6h_logical_block& log_blk); - + namespace bc7_mode_5_encoder { void encode_bc7_mode_5_block(void* pDst_block, color32* pPixels, bool hq_mode); } + namespace astc_6x6_hdr + { + extern uint8_t g_quantize_tables_preserve2[21 - 1][256]; // astc_helpers::TOTAL_ISE_RANGES=21 + extern uint8_t g_quantize_tables_preserve3[21 - 1][256]; + } // namespace astc_6x6_hdr + +#if BASISD_SUPPORT_XUASTC + namespace astc_ldr_t + { + const uint32_t ARITH_HEADER_MARKER = 0x01; + const uint32_t ARITH_HEADER_MARKER_BITS = 5; + + const uint32_t FULL_ZSTD_HEADER_MARKER = 0x01; + const uint32_t FULL_ZSTD_HEADER_MARKER_BITS = 5; + + const uint32_t FINAL_SYNC_MARKER = 0xAF; + const uint32_t FINAL_SYNC_MARKER_BITS = 8; + + const uint32_t cMaxConfigReuseNeighbors = 3; + +#pragma pack(push, 1) + struct xuastc_ldr_arith_header + { + uint8_t m_flags; + basisu::packed_uint<4> m_arith_bytes_len; + basisu::packed_uint<4> m_mean0_bits_len; + basisu::packed_uint<4> m_mean1_bytes_len; + basisu::packed_uint<4> m_run_bytes_len; + basisu::packed_uint<4> m_coeff_bytes_len; + basisu::packed_uint<4> m_sign_bits_len; + basisu::packed_uint<4> m_weight2_bits_len; // 2-bit weights (4 per byte), up to BISE_4_LEVELS + basisu::packed_uint<4> m_weight3_bits_len; // 3-bit weights (2 per byte), up to BISE_8_LEVELS + basisu::packed_uint<4> m_weight4_bits_len; // 4-bit weights (2 per byte), up to BISE_16_LEVELS + basisu::packed_uint<4> m_weight8_bytes_len; // 8-bit weights (1 per byte), up to BISE_32_LEVELS + basisu::packed_uint<4> m_unused; // Future expansion + }; + + struct xuastc_ldr_full_zstd_header + { + uint8_t m_flags; + + // Control + basisu::packed_uint<4> m_raw_bits_len; // uncompressed + basisu::packed_uint<4> m_mode_bytes_len; + basisu::packed_uint<4> m_solid_dpcm_bytes_len; + + // Endpoint DPCM + basisu::packed_uint<4> m_endpoint_dpcm_reuse_indices_len; + basisu::packed_uint<4> m_use_bc_bits_len; + basisu::packed_uint<4> m_endpoint_dpcm_3bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_4bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_5bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_6bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_7bit_len; + basisu::packed_uint<4> m_endpoint_dpcm_8bit_len; + + // Weight grid DCT + basisu::packed_uint<4> m_mean0_bits_len; + basisu::packed_uint<4> m_mean1_bytes_len; + basisu::packed_uint<4> m_run_bytes_len; + basisu::packed_uint<4> m_coeff_bytes_len; + basisu::packed_uint<4> m_sign_bits_len; + + // Weight DPCM + basisu::packed_uint<4> m_weight2_bits_len; // 2-bit weights (4 per byte), up to BISE_4_LEVELS + basisu::packed_uint<4> m_weight3_bits_len; // 3-bit weights (4 per byte), up to BISE_8_LEVELS + basisu::packed_uint<4> m_weight4_bits_len; // 4-bit weights (2 per byte), up to BISE_16_LEVELS + basisu::packed_uint<4> m_weight8_bytes_len; // 8-bit weights (1 per byte), up to BISE_32_LEVELS + + basisu::packed_uint<4> m_unused; // Future expansion + }; +#pragma pack(pop) + + const uint32_t DCT_RUN_LEN_EOB_SYM_INDEX = 64; + const uint32_t DCT_MAX_ARITH_COEFF_MAG = 255; + + const uint32_t DCT_MEAN_LEVELS0 = 9, DCT_MEAN_LEVELS1 = 33; + + const uint32_t PART_HASH_BITS = 6u; + const uint32_t PART_HASH_SIZE = 1u << PART_HASH_BITS; + + const uint32_t TM_HASH_BITS = 7u; + const uint32_t TM_HASH_SIZE = 1u << TM_HASH_BITS; + + typedef basisu::vector fvec; + + void init(); + + color_rgba blue_contract_enc(color_rgba orig, bool& did_clamp, int encoded_b); + color_rgba blue_contract_dec(int enc_r, int enc_g, int enc_b, int enc_a); + + struct astc_block_grid_config + { + uint16_t m_block_width, m_block_height; + uint16_t m_grid_width, m_grid_height; + + astc_block_grid_config() {} + + astc_block_grid_config(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height) + { + assert((block_width >= 4) && (block_width <= 12)); + assert((block_height >= 4) && (block_height <= 12)); + m_block_width = (uint16_t)block_width; + m_block_height = (uint16_t)block_height; + + assert((grid_width >= 2) && (grid_width <= block_width)); + assert((grid_height >= 2) && (grid_height <= block_height)); + m_grid_width = (uint16_t)grid_width; + m_grid_height = (uint16_t)grid_height; + } + + bool operator==(const astc_block_grid_config& other) const + { + return (m_block_width == other.m_block_width) && (m_block_height == other.m_block_height) && + (m_grid_width == other.m_grid_width) && (m_grid_height == other.m_grid_height); + } + }; + + struct astc_block_grid_data + { + float m_weight_gamma; + + // An unfortunate difference of containers, but in memory these matrices are both addressed as [r][c]. + basisu::vector2D m_upsample_matrix; + + basisu::vector m_downsample_matrix; + + astc_block_grid_data() {} + astc_block_grid_data(float weight_gamma) : m_weight_gamma(weight_gamma) {} + }; + + typedef basisu::hash_map > astc_block_grid_data_hash_t; + + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color32& l, color32& h); + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color32& l, color32& h, float* pScale = nullptr); + + void decode_endpoints_ise20(uint32_t cem_index, const uint8_t* pEndpoint_vals, color_rgba& l, color_rgba& h); + void decode_endpoints(uint32_t cem_index, const uint8_t* pEndpoint_vals, uint32_t endpoint_ise_index, color_rgba& l, color_rgba& h, float* pScale = nullptr); + + void compute_adjoint_downsample_matrix(basisu::vector& downsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + void compute_upsample_matrix(basisu::vector2D& upsample_matrix, uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + class dct2f + { + enum { cMaxSize = 12 }; + + public: + dct2f() : m_rows(0u), m_cols(0u) {} + + // call with grid_height/grid_width (INVERTED) + bool init(uint32_t rows, uint32_t cols); + + uint32_t rows() const { return m_rows; } + uint32_t cols() const { return m_cols; } + + void forward(const float* pSrc, float* pDst, fvec& work) const; + + void inverse(const float* pSrc, float* pDst, fvec& work) const; + + // check variants use a less optimized implementation, used for sanity checking + void inverse_check(const float* pSrc, float* pDst, fvec& work) const; + + void forward(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + void inverse(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + void inverse_check(const float* pSrc, uint32_t src_stride, + float* pDst, uint32_t dst_stride, fvec& work) const; + + private: + uint32_t m_rows, m_cols; + fvec m_c_col; // [u*m_rows + x] + fvec m_c_row; // [v*m_cols + y] + fvec m_a_col; // alpha(u) + fvec m_a_row; // alpha(v) + }; + + struct dct_syms + { + dct_syms() + { + clear(); + } + + void clear() + { + m_dc_sym = 0; + m_num_dc_levels = 0; + m_coeffs.resize(0); + m_max_coeff_mag = 0; + m_max_zigzag_index = 0; + } + + uint32_t m_dc_sym; + uint32_t m_num_dc_levels; + + struct coeff + { + uint16_t m_num_zeros; + int16_t m_coeff; // or INT16_MAX if invalid + + coeff() {} + coeff(uint16_t num_zeros, int16_t coeff) : m_num_zeros(num_zeros), m_coeff(coeff) {} + }; + + basisu::static_vector m_coeffs; + + uint32_t m_max_coeff_mag; + uint32_t m_max_zigzag_index; + }; + + struct grid_dim_key + { + int m_grid_width; + int m_grid_height; + + grid_dim_key() {} + + grid_dim_key(int w, int h) : m_grid_width(w), m_grid_height(h) {} + + bool operator== (const grid_dim_key& rhs) const + { + return (m_grid_width == rhs.m_grid_width) && (m_grid_height == rhs.m_grid_height); + } + }; + + struct grid_dim_value + { + basisu::int_vec m_zigzag; + dct2f m_dct; + }; + + typedef basisu::hash_map > grid_dim_hash_map; + + void init_astc_block_grid_data_hash(); + + const astc_block_grid_data* find_astc_block_grid_data(uint32_t block_width, uint32_t block_height, uint32_t grid_width, uint32_t grid_height); + + const float DEADZONE_ALPHA = .5f; + const float SCALED_WEIGHT_BASE_CODING_SCALE = .5f; // typically ~5 bits [0,32], or 3 [0,8] + + struct sample_quant_table_state + { + float m_q, m_sx, m_sy, m_level_scale; + + void init(float q, + uint32_t block_width, uint32_t block_height, + float level_scale) + { + m_q = q; + m_level_scale = level_scale; + + const int Bx = block_width, By = block_height; + + m_sx = (float)8.0f / (float)Bx; + m_sy = (float)8.0f / (float)By; + } + }; + + class grid_weight_dct + { + public: + grid_weight_dct() { } + + void init(uint32_t block_width, uint32_t block_height); + + static uint32_t get_num_weight_dc_levels(uint32_t weight_ise_range) + { + float scaled_weight_coding_scale = SCALED_WEIGHT_BASE_CODING_SCALE; + if (weight_ise_range <= astc_helpers::BISE_8_LEVELS) + scaled_weight_coding_scale = 1.0f / 8.0f; + + return (uint32_t)(64.0f * scaled_weight_coding_scale) + 1; + } + + struct block_stats + { + float m_mean_weight; + uint32_t m_total_coded_acs; + uint32_t m_max_ac_coeff; + }; + + bool decode_block_weights( + float q, uint32_t plane_index, // plane of weights to decode and IDCT from stream + astc_helpers::log_astc_block& log_blk, // must be initialized except for the plane weights which are decoded + basist::bitwise_decoder* pDec, + const astc_block_grid_data* pGrid_data, // grid data for this grid size + block_stats* pS, + fvec& dct_work, // thread local + const dct_syms* pSyms = nullptr) const; + + enum { m_zero_run = 3, m_coeff = 2 }; + + uint32_t m_block_width, m_block_height; + + grid_dim_hash_map m_grid_dim_key_vals; + + // Adaptively compensate for weight level quantization noise being fed into the DCT. + // The more coursely the weight levels are quantized, the more noise injected, and the more noise will be spread between multiple AC coefficients. + // This will cause some previously 0 coefficients to increase in mag, but they're likely noise. So carefully nudge the quant step size to compensate. + static float scale_quant_steps(int Q_astc, float gamma = 0.1f /*.13f*/, float clamp_max = 2.0f) + { + assert(Q_astc >= 2); + float factor = 63.0f / (Q_astc - 1); + // TODO: Approximate powf() + float scaled = powf(factor, gamma); + scaled = basisu::clamp(scaled, 1.0f, clamp_max); + return scaled; + } + + float compute_level_scale(float q, float span_len, float weight_gamma, uint32_t grid_width, uint32_t grid_height, uint32_t weight_ise_range) const; + + int sample_quant_table(sample_quant_table_state& state, uint32_t x, uint32_t y) const; + + void compute_quant_table(float q, + uint32_t grid_width, uint32_t grid_height, + float level_scale, int* dct_quant_tab) const; + + float get_max_span_len(const astc_helpers::log_astc_block& log_blk, uint32_t plane_index) const; + + inline int quantize_deadzone(float d, int L, float alpha, uint32_t x, uint32_t y) const + { + assert((x < m_block_width) && (y < m_block_height)); + + if (((x == 1) && (y == 0)) || + ((x == 0) && (y == 1))) + { + return (int)std::round(d / (float)L); + } + + // L = quant step, alpha in [0,1.2] (typical 0.7–0.85) + if (L <= 0) + return 0; + + float s = fabsf(d); + float tau = alpha * float(L); // half-width of the zero band + + if (s <= tau) + return 0; // inside dead-zone towards zero + + // Quantize the residual outside the dead-zone with mid-tread rounding + float qf = (s - tau) / float(L); + int q = (int)floorf(qf + 0.5f); // ties-nearest + return (d < 0.0f) ? -q : q; + } + + inline float dequant_deadzone(int q, int L, float alpha, uint32_t x, uint32_t y) const + { + assert((x < m_block_width) && (y < m_block_height)); + + if (((x == 1) && (y == 0)) || + ((x == 0) && (y == 1))) + { + return (float)q * (float)L; + } + + if (q == 0 || L <= 0) + return 0.0f; + + float tau = alpha * float(L); + float mag = tau + float(abs(q)) * float(L); // center of the (nonzero) bin + return (q < 0) ? -mag : mag; + } + }; + + struct trial_mode + { + uint32_t m_grid_width; + uint32_t m_grid_height; + uint32_t m_cem; + int m_ccs_index; + uint32_t m_endpoint_ise_range; + uint32_t m_weight_ise_range; + uint32_t m_num_parts; + + bool operator==(const trial_mode& other) const + { +#define BU_COMP(a) if (a != other.a) return false; + BU_COMP(m_grid_width); + BU_COMP(m_grid_height); + BU_COMP(m_cem); + BU_COMP(m_ccs_index); + BU_COMP(m_endpoint_ise_range); + BU_COMP(m_weight_ise_range); + BU_COMP(m_num_parts); +#undef BU_COMP + return true; + } + + bool operator<(const trial_mode& rhs) const + { +#define BU_COMP(a) if (a < rhs.a) return true; else if (a > rhs.a) return false; + BU_COMP(m_grid_width); + BU_COMP(m_grid_height); + BU_COMP(m_cem); + BU_COMP(m_ccs_index); + BU_COMP(m_endpoint_ise_range); + BU_COMP(m_weight_ise_range); + BU_COMP(m_num_parts); +#undef BU_COMP + return false; + } + + operator size_t() const + { + size_t h = 0xABC1F419; +#define BU_FIELD(a) do { h ^= hash_hsieh(reinterpret_cast(&a), sizeof(a)); } while(0) + BU_FIELD(m_grid_width); + BU_FIELD(m_grid_height); + BU_FIELD(m_cem); + BU_FIELD(m_ccs_index); + BU_FIELD(m_endpoint_ise_range); + BU_FIELD(m_weight_ise_range); + BU_FIELD(m_num_parts); +#undef BU_FIELD + return h; + } + }; + + // Organize trial modes for faster initial mode triaging. + const uint32_t OTM_NUM_CEMS = 14; // 0-13 (13=highest valid LDR CEM) + const uint32_t OTM_NUM_SUBSETS = 3; // 1-3 + const uint32_t OTM_NUM_CCS = 5; // -1 to 3 + const uint32_t OTM_NUM_GRID_SIZES = 2; // 0=small or 1=large (grid_w>=block_w-1 and grid_h>=block_h-1) + const uint32_t OTM_NUM_GRID_ANISOS = 3; // 0=W=H, 1=W>H, 2=W 0) && (gh > 0)); + assert((bw > 0) && (bh > 0)); + assert((gw <= 12) && (gh <= 12) && (bw <= 12) && (bh <= 12)); + assert((gw <= bw) && (gh <= bh)); + +#if 0 + // Prev. code: + uint32_t grid_aniso = 0; + if (tm.m_grid_width != tm.m_grid_height) // not optimal for non-square block sizes + { + const float grid_x_fract = (float)tm.m_grid_width / (float)block_width; + const float grid_y_fract = (float)tm.m_grid_height / (float)block_height; + if (grid_x_fract >= grid_y_fract) + grid_aniso = 1; + else if (grid_x_fract < grid_y_fract) + grid_aniso = 2; + } +#endif + // Compare gw/bw vs. gh/bh using integer math: + // gw*bh >= gh*bw -> X-dominant (1), else Y-dominant (2) + const uint32_t lhs = gw * bh; + const uint32_t rhs = gh * bw; + + // Equal (isotropic), X=Y + if (lhs == rhs) + return 0; + + // Anisotropic - 1=X, 2=Y + return (lhs >= rhs) ? 1 : 2; + } + + struct grouped_trial_modes + { + basisu::uint_vec m_tm_groups[OTM_NUM_CEMS][OTM_NUM_SUBSETS][OTM_NUM_CCS][OTM_NUM_GRID_SIZES][OTM_NUM_GRID_ANISOS]; // indices of encoder trial modes in each bucket + + void clear() + { + for (uint32_t cem_iter = 0; cem_iter < OTM_NUM_CEMS; cem_iter++) + for (uint32_t subsets_iter = 0; subsets_iter < OTM_NUM_SUBSETS; subsets_iter++) + for (uint32_t ccs_iter = 0; ccs_iter < OTM_NUM_CCS; ccs_iter++) + for (uint32_t grid_sizes_iter = 0; grid_sizes_iter < OTM_NUM_GRID_SIZES; grid_sizes_iter++) + for (uint32_t grid_anisos_iter = 0; grid_anisos_iter < OTM_NUM_GRID_ANISOS; grid_anisos_iter++) + m_tm_groups[cem_iter][subsets_iter][ccs_iter][grid_sizes_iter][grid_anisos_iter].clear(); + } + + void add(uint32_t block_width, uint32_t block_height, + const trial_mode& tm, uint32_t tm_index) + { + const uint32_t cem_index = tm.m_cem; + assert(cem_index < OTM_NUM_CEMS); + + const uint32_t subset_index = tm.m_num_parts - 1; + assert(subset_index < OTM_NUM_SUBSETS); + + const uint32_t ccs_index = tm.m_ccs_index + 1; + assert(ccs_index < OTM_NUM_CCS); + + const uint32_t grid_size = (tm.m_grid_width >= (block_width - 1)) && (tm.m_grid_height >= (block_height - 1)); + const uint32_t grid_aniso = calc_grid_aniso_val(tm.m_grid_width, tm.m_grid_height, block_width, block_height); + + basisu::uint_vec& v = m_tm_groups[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + if (!v.capacity()) + v.reserve(64); + + v.push_back(tm_index); + } + + uint32_t count_used_groups() const + { + uint32_t n = 0; + + for (uint32_t cem_iter = 0; cem_iter < OTM_NUM_CEMS; cem_iter++) + for (uint32_t subsets_iter = 0; subsets_iter < OTM_NUM_SUBSETS; subsets_iter++) + for (uint32_t ccs_iter = 0; ccs_iter < OTM_NUM_CCS; ccs_iter++) + for (uint32_t grid_sizes_iter = 0; grid_sizes_iter < OTM_NUM_GRID_SIZES; grid_sizes_iter++) + for (uint32_t grid_anisos_iter = 0; grid_anisos_iter < OTM_NUM_GRID_ANISOS; grid_anisos_iter++) + { + if (m_tm_groups[cem_iter][subsets_iter][ccs_iter][grid_sizes_iter][grid_anisos_iter].size()) + n++; + } + return n; + } + }; + + extern grouped_trial_modes g_grouped_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + inline const basisu::uint_vec& get_tm_candidates(const grouped_trial_modes& grouped_enc_trial_modes, + uint32_t cem_index, uint32_t subset_index, uint32_t ccs_index, uint32_t grid_size, uint32_t grid_aniso) + { + assert(cem_index < OTM_NUM_CEMS); + assert(subset_index < OTM_NUM_SUBSETS); + assert(ccs_index < OTM_NUM_CCS); + assert(grid_size < OTM_NUM_GRID_SIZES); + assert(grid_aniso < OTM_NUM_GRID_ANISOS); + + const basisu::uint_vec& modes = grouped_enc_trial_modes.m_tm_groups[cem_index][subset_index][ccs_index][grid_size][grid_aniso]; + return modes; + } + + const uint32_t CFG_PACK_GRID_BITS = 7; + const uint32_t CFG_PACK_CEM_BITS = 3; + const uint32_t CFG_PACK_CCS_BITS = 3; + const uint32_t CFG_PACK_SUBSETS_BITS = 2; + const uint32_t CFG_PACK_WISE_BITS = 4; + const uint32_t CFG_PACK_EISE_BITS = 5; + + extern const int s_unique_ldr_index_to_astc_cem[6]; + + enum class xuastc_mode + { + cMODE_SOLID = 0, + cMODE_RAW = 1, + + // Full cfg, partition ID, and all endpoint value reuse. + cMODE_REUSE_CFG_ENDPOINTS_LEFT = 2, + cMODE_REUSE_CFG_ENDPOINTS_UP = 3, + cMODE_REUSE_CFG_ENDPOINTS_DIAG = 4, + + cMODE_RUN = 5, + + cMODE_TOTAL, + }; + + enum class xuastc_zstd_mode + { + // len=1 bits + cMODE_RAW = 0b0, + + // len=2 bits + cMODE_RUN = 0b01, + + // len=4 bits + cMODE_SOLID = 0b0011, + cMODE_REUSE_CFG_ENDPOINTS_LEFT = 0b0111, + cMODE_REUSE_CFG_ENDPOINTS_UP = 0b1011, + cMODE_REUSE_CFG_ENDPOINTS_DIAG = 0b1111 + }; + + const uint32_t XUASTC_LDR_MODE_BYTE_IS_BASE_OFS_FLAG = 1 << 3; + const uint32_t XUASTC_LDR_MODE_BYTE_PART_HASH_HIT = 1 << 4; + const uint32_t XUASTC_LDR_MODE_BYTE_DPCM_ENDPOINTS_FLAG = 1 << 5; + const uint32_t XUASTC_LDR_MODE_BYTE_TM_HASH_HIT_FLAG = 1 << 6; + const uint32_t XUASTC_LDR_MODE_BYTE_USE_DCT = 1 << 7; + + enum class xuastc_ldr_syntax + { + cFullArith = 0, + cHybridArithZStd = 1, + cFullZStd = 2, + + cTotal + }; + + void create_encoder_trial_modes_table(uint32_t block_width, uint32_t block_height, + basisu::vector& encoder_trial_modes, grouped_trial_modes& grouped_encoder_trial_modes, + bool print_debug_info, bool print_modes); + + extern basisu::vector g_encoder_trial_modes[astc_helpers::cTOTAL_BLOCK_SIZES]; + + inline uint32_t part_hash_index(uint32_t x) + { + // fib hash + return (x * 2654435769u) & (PART_HASH_SIZE - 1); + } + + // Full ZStd syntax only + inline uint32_t tm_hash_index(uint32_t x) + { + // fib hash + return (x * 2654435769u) & (TM_HASH_SIZE - 1); + } + + // TODO: Some fields are unused during transcoding. + struct prev_block_state + { + bool m_was_solid_color; + bool m_used_weight_dct; + bool m_first_endpoint_uses_bc; + bool m_reused_full_cfg; + bool m_used_part_hash; + + int m_tm_index; // -1 if invalid (solid color block) + uint32_t m_base_cem_index; // doesn't include base+ofs + uint32_t m_subset_index, m_ccs_index, m_grid_size, m_grid_aniso; + + prev_block_state() + { + clear(); + } + + void clear() + { + basisu::clear_obj(*this); + } + }; + + struct prev_block_state_full_zstd + { + int m_tm_index; // -1 if invalid (solid color block) + + bool was_solid_color() const { return m_tm_index < 0; } + + prev_block_state_full_zstd() + { + clear(); + } + + void clear() + { + basisu::clear_obj(*this); + } + }; + + inline uint32_t cem_to_ldrcem_index(uint32_t cem) + { + switch (cem) + { + case astc_helpers::CEM_LDR_LUM_DIRECT: return 0; + case astc_helpers::CEM_LDR_LUM_ALPHA_DIRECT: return 1; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE: return 2; + case astc_helpers::CEM_LDR_RGB_DIRECT: return 3; + case astc_helpers::CEM_LDR_RGB_BASE_PLUS_OFFSET: return 4; + case astc_helpers::CEM_LDR_RGB_BASE_SCALE_PLUS_TWO_A: return 5; + case astc_helpers::CEM_LDR_RGBA_DIRECT: return 6; + case astc_helpers::CEM_LDR_RGBA_BASE_PLUS_OFFSET: return 7; + default: + assert(0); + break; + } + + return 0; + } + + bool pack_base_offset( + uint32_t cem_index, uint32_t dst_ise_endpoint_range, uint8_t* pPacked_endpoints, + const color_rgba& l, const color_rgba& h, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag, bool& endpoints_swapped); + + bool convert_endpoints_across_cems( + uint32_t prev_cem, uint32_t prev_endpoint_ise_range, const uint8_t* pPrev_endpoints, + uint32_t dst_cem, uint32_t dst_endpoint_ise_range, uint8_t* pDst_endpoints, + bool always_repack, + bool use_blue_contraction, bool auto_disable_blue_contraction_if_clamped, + bool& blue_contraction_clamped_flag, bool& base_ofs_clamped_flag); + + uint32_t get_total_unique_patterns(uint32_t astc_block_size_index, uint32_t num_parts); + //uint16_t unique_pat_index_to_part_seed(uint32_t astc_block_size_index, uint32_t num_parts, uint32_t unique_pat_index); + + typedef bool (*xuastc_decomp_image_init_callback_ptr)(uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t block_width, uint32_t block_height, bool srgb_decode_profile, float dct_q, bool has_alpha, void* pData); + typedef bool (*xuastc_decomp_image_block_callback_ptr)(uint32_t bx, uint32_t by, const astc_helpers::log_astc_block& log_blk, void* pData); + + bool xuastc_ldr_decompress_image( + const uint8_t* pComp_data, size_t comp_data_size, + uint32_t& astc_block_width, uint32_t& astc_block_height, + uint32_t& actual_width, uint32_t& actual_height, bool& has_alpha, bool& uses_srgb_astc_decode_mode, + bool debug_output, + xuastc_decomp_image_init_callback_ptr pInit_callback, void *pInit_callback_data, + xuastc_decomp_image_block_callback_ptr pBlock_callback, void *pBlock_callback_data); + + } // namespace astc_ldr_t + + namespace arith_fastbits_f32 + { + enum { TABLE_BITS = 8 }; // 256..1024 entries typical (8..10) + enum { TABLE_SIZE = 1 << TABLE_BITS }; + enum { MANT_BITS = 23 }; + enum { FRAC_BITS = MANT_BITS - TABLE_BITS }; + enum { FRAC_MASK = (1u << FRAC_BITS) - 1u }; + + extern bool g_initialized; + extern float g_lut_edge[TABLE_SIZE + 1]; // samples at m = 1 + i/TABLE_SIZE (for linear) + + inline void init() + { + if (g_initialized) + return; + + const float inv_ln2 = 1.4426950408889634f; // 1/ln(2) + + for (int i = 0; i <= TABLE_SIZE; ++i) + { + float m = 1.0f + float(i) / float(TABLE_SIZE); // m in [1,2] + g_lut_edge[i] = logf(m) * inv_ln2; // log2(m) + } + + g_initialized = true; + } + + inline void unpack(float p, int& e_unbiased, uint32_t& mant) + { + // kill any denorms + if (p < FLT_MIN) + p = 0; + + union { float f; uint32_t u; } x; + x.f = p; + e_unbiased = int((x.u >> 23) & 0xFF) - 127; + mant = (x.u & 0x7FFFFFu); // 23-bit mantissa + } + + // Returns estimated bits given probability p, approximates -log2f(p). + inline float bits_from_prob_linear(float p) + { + assert((p > 0.0f) && (p <= 1.0f)); + if (!g_initialized) + init(); + + int e; uint32_t mant; + unpack(p, e, mant); + + uint32_t idx = mant >> FRAC_BITS; // 0..TABLE_SIZE-1 + uint32_t frac = mant & FRAC_MASK; // low FRAC_BITS + const float inv_scale = 1.0f / float(1u << FRAC_BITS); + float t = float(frac) * inv_scale; // [0,1) + + float y0 = g_lut_edge[idx]; + float y1 = g_lut_edge[idx + 1]; + float log2m = y0 + t * (y1 - y0); + + return -(float(e) + log2m); + } + + } // namespace arith_fastbits_f32 + + namespace arith + { + // A simple range coder + const uint32_t ArithMaxSyms = 2048; + const uint32_t DMLenShift = 15u; + const uint32_t DMMaxCount = 1u << DMLenShift; + const uint32_t BMLenShift = 13u; + const uint32_t BMMaxCount = 1u << BMLenShift; + const uint32_t ArithMinLen = 1u << 24u; + const uint32_t ArithMaxLen = UINT32_MAX; + const uint32_t ArithMinExpectedDataBufSize = 5; + + class arith_bit_model + { + public: + arith_bit_model() + { + reset(); + } + + void init() + { + reset(); + } + + void reset() + { + m_bit0_count = 1; + m_bit_count = 2; + m_bit0_prob = 1U << (BMLenShift - 1); + m_update_interval = 4; + m_bits_until_update = 4; + } + + float get_price(bool bit) const + { + const float prob_0 = (float)m_bit0_prob / (float)BMMaxCount; + const float prob = bit ? (1.0f - prob_0) : prob_0; + const float bits = arith_fastbits_f32::bits_from_prob_linear(prob); + assert(fabs(bits - (-log2f(prob))) < .00125f); // basic sanity check + return bits; + } + + void update() + { + assert(m_bit_count >= 2); + assert(m_bit0_count < m_bit_count); + + if (m_bit_count >= BMMaxCount) + { + assert(m_bit_count && m_bit0_count); + + m_bit_count = (m_bit_count + 1) >> 1; + m_bit0_count = (m_bit0_count + 1) >> 1; + + if (m_bit0_count == m_bit_count) + ++m_bit_count; + + assert(m_bit0_count < m_bit_count); + } + + const uint32_t scale = 0x80000000U / m_bit_count; + m_bit0_prob = (m_bit0_count * scale) >> (31 - BMLenShift); + + m_update_interval = basisu::clamp((5 * m_update_interval) >> 2, 4u, 128); + + m_bits_until_update = m_update_interval; + } + + void print_prices(const char* pDesc) + { + if (pDesc) + printf("arith_data_model bit prices for model %s:\n", pDesc); + for (uint32_t i = 0; i < 2; i++) + printf("%u: %3.3f bits\n", i, get_price(i)); + printf("\n"); + } + + private: + friend class arith_enc; + friend class arith_dec; + + uint32_t m_bit0_prob; // snapshot made at last update + + uint32_t m_bit0_count; // live + uint32_t m_bit_count; // live + + int m_bits_until_update; + uint32_t m_update_interval; + }; + + enum { cARITH_GAMMA_MAX_TAIL_CTX = 4, cARITH_GAMMA_MAX_PREFIX_CTX = 3 }; + struct arith_gamma_contexts + { + arith_bit_model m_ctx_prefix[cARITH_GAMMA_MAX_PREFIX_CTX]; // for unary continue prefix + arith_bit_model m_ctx_tail[cARITH_GAMMA_MAX_TAIL_CTX]; // for binary suffix bits + }; + + class arith_data_model + { + public: + arith_data_model() : + m_num_data_syms(0), + m_total_sym_freq(0), + m_update_interval(0), + m_num_syms_until_next_update(0) + { + } + + arith_data_model(uint32_t num_syms, bool faster_update = false) : + m_num_data_syms(0), + m_total_sym_freq(0), + m_update_interval(0), + m_num_syms_until_next_update(0) + { + init(num_syms, faster_update); + } + + void clear() + { + m_cum_sym_freqs.clear(); + m_sym_freqs.clear(); + + m_num_data_syms = 0; + m_total_sym_freq = 0; + m_update_interval = 0; + m_num_syms_until_next_update = 0; + } + + void init(uint32_t num_syms, bool faster_update = false) + { + assert((num_syms >= 2) && (num_syms <= ArithMaxSyms)); + + m_num_data_syms = num_syms; + + m_sym_freqs.resize(num_syms); + m_cum_sym_freqs.resize(num_syms + 1); + + reset(faster_update); + } + + void reset(bool faster_update = false) + { + if (!m_num_data_syms) + return; + + m_sym_freqs.set_all(1); + m_total_sym_freq = m_num_data_syms; + + m_update_interval = m_num_data_syms; + m_num_syms_until_next_update = 0; + + update(false); + + if (faster_update) + { + m_update_interval = basisu::clamp((m_num_data_syms + 7) / 8, 4u, (m_num_data_syms + 6) << 3); + m_num_syms_until_next_update = m_update_interval; + } + } + + void update(bool enc_flag) + { + assert(m_num_data_syms); + BASISU_NOTE_UNUSED(enc_flag); + + if (!m_num_data_syms) + return; + + while (m_total_sym_freq >= DMMaxCount) + { + m_total_sym_freq = 0; + + for (uint32_t n = 0; n < m_num_data_syms; n++) + { + m_sym_freqs[n] = (m_sym_freqs[n] + 1u) >> 1u; + m_total_sym_freq += m_sym_freqs[n]; + } + } + + const uint32_t scale = 0x80000000U / m_total_sym_freq; + + uint32_t sum = 0; + for (uint32_t i = 0; i < m_num_data_syms; ++i) + { + assert(((uint64_t)scale * sum) <= UINT32_MAX); + m_cum_sym_freqs[i] = (scale * sum) >> (31 - DMLenShift); + sum += m_sym_freqs[i]; + } + assert(sum == m_total_sym_freq); + + m_cum_sym_freqs[m_num_data_syms] = DMMaxCount; + + m_update_interval = basisu::clamp((5 * m_update_interval) >> 2, 4u, (m_num_data_syms + 6) << 3); + + m_num_syms_until_next_update = m_update_interval; + } + + float get_price(uint32_t sym_index) const + { + assert(sym_index < m_num_data_syms); + + if (sym_index >= m_num_data_syms) + return 0.0f; + + const float prob = (float)(m_cum_sym_freqs[sym_index + 1] - m_cum_sym_freqs[sym_index]) / (float)DMMaxCount; + + const float bits = arith_fastbits_f32::bits_from_prob_linear(prob); + assert(fabs(bits - (-log2f(prob))) < .00125f); // basic sanity check + return bits; + } + + void print_prices(const char* pDesc) + { + if (pDesc) + printf("arith_data_model bit prices for model %s:\n", pDesc); + for (uint32_t i = 0; i < m_num_data_syms; i++) + printf("%u: %3.3f bits\n", i, get_price(i)); + printf("\n"); + } + + uint32_t get_num_data_syms() const { return m_num_data_syms; } + + private: + friend class arith_enc; + friend class arith_dec; + + uint32_t m_num_data_syms; + + basisu::uint_vec m_sym_freqs; // live histogram + uint32_t m_total_sym_freq; // always live vs. m_sym_freqs + + basisu::uint_vec m_cum_sym_freqs; // has 1 extra entry, snapshot from last update + + uint32_t m_update_interval; + int m_num_syms_until_next_update; + + uint32_t get_last_sym_index() const { return m_num_data_syms - 1; } + }; + + class arith_enc + { + public: + arith_enc() + { + clear(); + } + + void clear() + { + m_data_buf.clear(); + + m_base = 0; + m_length = ArithMaxLen; + } + + void init(size_t reserve_size) + { + m_data_buf.reserve(reserve_size); + m_data_buf.resize(0); + + m_base = 0; + m_length = ArithMaxLen; + + // Place 8-bit marker at beginning. + // This virtually always guarantees no backwards carries can be lost at the very beginning of the stream. (Should be impossible with this design.) + // It always pushes out 1 0 byte at the very beginning to absorb future carries. + // Caller does this now, we send a tiny header anyway + //put_bits(0x1, 8); + //assert(m_data_buf[0] != 0xFF); + } + + void put_bit(uint32_t bit) + { + m_length >>= 1; + + if (bit) + { + const uint32_t orig_base = m_base; + + m_base += m_length; + + if (orig_base > m_base) + prop_carry(); + } + + if (m_length < ArithMinLen) + renorm(); + } + + enum { cMaxPutBitsLen = 20 }; + void put_bits(uint32_t val, uint32_t num_bits) + { + assert(num_bits && (num_bits <= cMaxPutBitsLen)); + assert(val < (1u << num_bits)); + + m_length >>= num_bits; + + const uint32_t orig_base = m_base; + + m_base += val * m_length; + + if (orig_base > m_base) + prop_carry(); + + if (m_length < ArithMinLen) + renorm(); + } + + // returns # of bits actually written + inline uint32_t put_truncated_binary(uint32_t v, uint32_t n) + { + assert((n >= 2) && (v < n)); + + uint32_t k = basisu::floor_log2i(n); + uint32_t u = (1 << (k + 1)) - n; + + if (v < u) + { + put_bits(v, k); + return k; + } + + uint32_t x = v + u; + assert((x >> 1) >= u); + + put_bits(x >> 1, k); + put_bits(x & 1, 1); + return k + 1; + } + + static inline uint32_t get_truncated_binary_bits(uint32_t v, uint32_t n) + { + assert((n >= 2) && (v < n)); + + uint32_t k = basisu::floor_log2i(n); + uint32_t u = (1 << (k + 1)) - n; + + if (v < u) + return k; + +#ifdef _DEBUG + uint32_t x = v + u; + assert((x >> 1) >= u); +#endif + + return k + 1; + } + + inline uint32_t put_rice(uint32_t v, uint32_t m) + { + assert(m); + + uint32_t q = v >> m, r = v & ((1 << m) - 1); + + // rice coding sanity check + assert(q <= 64); + + uint32_t total_bits = q; + + // TODO: put_bits the pattern inverted in bit order + while (q) + { + put_bit(1); + q--; + } + + put_bit(0); + + put_bits(r, m); + + total_bits += (m + 1); + + return total_bits; + } + + static inline uint32_t get_rice_price(uint32_t v, uint32_t m) + { + assert(m); + + uint32_t q = v >> m; + + // rice coding sanity check + assert(q <= 64); + + uint32_t total_bits = q + 1 + m; + + return total_bits; + } + + inline void put_gamma(uint32_t n, arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return; + } + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + encode(1, ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + encode(0, ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + encode(bit, ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + } + } + + inline float put_gamma_and_return_price(uint32_t n, arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return 0.0f; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return 0.0f; + } + + float total_price = 0.0f; + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + { + total_price += ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(1); + encode(1, ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + } + + total_price += ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(0); + encode(0, ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)]); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + total_price += ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)].get_price(bit); + encode(bit, ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + } + + return total_price; + } + + // prediced price, won't be accurate if a binary arith model decides to update in between + inline float get_gamma_price(uint32_t n, const arith_gamma_contexts& ctxs) + { + assert(n); + if (!n) + return 0.0f; + + const int k = basisu::floor_log2i(n); + if (k > 16) + { + assert(0); + return 0.0f; + } + + float total_price = 0.0f; + + // prefix: k times '1' then a '0' + for (int i = 0; i < k; ++i) + total_price += ctxs.m_ctx_prefix[basisu::minimum(i, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(1); + + total_price += ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)].get_price(0); + + // suffix: the k low bits of n + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = (n >> i) & 1u; + total_price += ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)].get_price(bit); + } + + return total_price; + } + + void encode(uint32_t bit, arith_bit_model& dm) + { + uint32_t x = dm.m_bit0_prob * (m_length >> BMLenShift); + + if (!bit) + { + m_length = x; + ++dm.m_bit0_count; + } + else + { + const uint32_t orig_base = m_base; + m_base += x; + m_length -= x; + + if (orig_base > m_base) + prop_carry(); + } + ++dm.m_bit_count; + + if (m_length < ArithMinLen) + renorm(); + + if (--dm.m_bits_until_update <= 0) + dm.update(); + } + + float encode_and_return_price(uint32_t bit, arith_bit_model& dm) + { + const float price = dm.get_price(bit); + encode(bit, dm); + return price; + } + + void encode(uint32_t sym, arith_data_model& dm) + { + assert(sym < dm.m_num_data_syms); + + const uint32_t orig_base = m_base; + + if (sym == dm.get_last_sym_index()) + { + uint32_t x = dm.m_cum_sym_freqs[sym] * (m_length >> DMLenShift); + m_base += x; + m_length -= x; + } + else + { + m_length >>= DMLenShift; + uint32_t x = dm.m_cum_sym_freqs[sym] * m_length; + m_base += x; + m_length = dm.m_cum_sym_freqs[sym + 1] * m_length - x; + } + + if (orig_base > m_base) + prop_carry(); + + if (m_length < ArithMinLen) + renorm(); + + ++dm.m_sym_freqs[sym]; + ++dm.m_total_sym_freq; + + if (--dm.m_num_syms_until_next_update <= 0) + dm.update(true); + } + + float encode_and_return_price(uint32_t sym, arith_data_model& dm) + { + const float price = dm.get_price(sym); + encode(sym, dm); + return price; + } + + void flush() + { + const uint32_t orig_base = m_base; + + if (m_length <= (2 * ArithMinLen)) + { + m_base += ArithMinLen >> 1; + m_length = ArithMinLen >> 9; + } + else + { + m_base += ArithMinLen; + m_length = ArithMinLen >> 1; + } + + if (orig_base > m_base) + prop_carry(); + + renorm(); + + // Pad output to min 5 bytes - quite conservative; we're typically compressing large streams so the overhead shouldn't matter. + if (m_data_buf.size() < ArithMinExpectedDataBufSize) + m_data_buf.resize(ArithMinExpectedDataBufSize); + } + + basisu::uint8_vec& get_data_buf() { return m_data_buf; } + const basisu::uint8_vec& get_data_buf() const { return m_data_buf; } + + private: + basisu::uint8_vec m_data_buf; + uint32_t m_base, m_length; + + inline void prop_carry() + { + int64_t ofs = m_data_buf.size() - 1; + + for (; (ofs >= 0) && (m_data_buf[(size_t)ofs] == 0xFF); --ofs) + m_data_buf[(size_t)ofs] = 0; + + if (ofs >= 0) + ++m_data_buf[(size_t)ofs]; + } + + inline void renorm() + { + assert(m_length < ArithMinLen); + do + { + m_data_buf.push_back((uint8_t)(m_base >> 24u)); + m_base <<= 8u; + m_length <<= 8u; + } while (m_length < ArithMinLen); + } + }; + + class arith_dec + { + public: + arith_dec() + { + clear(); + } + + void clear() + { + m_pData_buf = nullptr; + m_pData_buf_last_byte = nullptr; + m_pData_buf_cur = nullptr; + m_data_buf_size = 0; + + m_value = 0; + m_length = 0; + } + + bool init(const uint8_t* pBuf, size_t buf_size) + { + if (buf_size < ArithMinExpectedDataBufSize) + { + assert(0); + return false; + } + + m_pData_buf = pBuf; + m_pData_buf_last_byte = pBuf + buf_size - 1; + m_pData_buf_cur = m_pData_buf + 4; + m_data_buf_size = buf_size; + + m_value = ((uint32_t)(pBuf[0]) << 24u) | ((uint32_t)(pBuf[1]) << 16u) | ((uint32_t)(pBuf[2]) << 8u) | (uint32_t)(pBuf[3]); + m_length = ArithMaxLen; + + // Check for the 8-bit marker we always place at the beginning of the stream. + //uint32_t marker = get_bits(8); + //if (marker != 0x1) + // return false; + + return true; + } + + uint32_t get_bit() + { + assert(m_data_buf_size); + + m_length >>= 1; + + uint32_t bit = (m_value >= m_length); + + if (bit) + m_value -= m_length; + + if (m_length < ArithMinLen) + renorm(); + + return bit; + } + + enum { cMaxGetBitsLen = 20 }; + + uint32_t get_bits(uint32_t num_bits) + { + assert(m_data_buf_size); + + if ((num_bits < 1) || (num_bits > cMaxGetBitsLen)) + { + assert(0); + return 0; + } + + m_length >>= num_bits; + assert(m_length); + + const uint32_t v = m_value / m_length; + + m_value -= m_length * v; + + if (m_length < ArithMinLen) + renorm(); + + return v; + } + + uint32_t decode_truncated_binary(uint32_t n) + { + assert(n >= 2); + + const uint32_t k = basisu::floor_log2i(n); + const uint32_t u = (1 << (k + 1)) - n; + + uint32_t result = get_bits(k); + + if (result >= u) + result = ((result << 1) | get_bits(1)) - u; + + return result; + } + + uint32_t decode_rice(uint32_t m) + { + assert(m); + + uint32_t q = 0; + for (;;) + { + uint32_t k = get_bit(); + if (!k) + break; + + q++; + if (q > 64) + { + assert(0); + return 0; + } + } + + return (q << m) + get_bits(m); + } + + uint32_t decode_bit(arith_bit_model& dm) + { + assert(m_data_buf_size); + + uint32_t x = dm.m_bit0_prob * (m_length >> BMLenShift); + uint32_t bit = (m_value >= x); + + if (bit == 0) + { + m_length = x; + ++dm.m_bit0_count; + } + else + { + m_value -= x; + m_length -= x; + } + ++dm.m_bit_count; + + if (m_length < ArithMinLen) + renorm(); + + if (--dm.m_bits_until_update <= 0) + dm.update(); + + return bit; + } + + inline uint32_t decode_gamma(arith_gamma_contexts& ctxs) + { + int k = 0; + while (decode_bit(ctxs.m_ctx_prefix[basisu::minimum(k, cARITH_GAMMA_MAX_PREFIX_CTX - 1)])) + { + ++k; + + if (k > 16) + { + // something is very wrong + assert(0); + return 0; + } + } + + int n = 1 << k; + for (int i = k - 1; i >= 0; --i) + { + uint32_t bit = decode_bit(ctxs.m_ctx_tail[basisu::minimum(i, cARITH_GAMMA_MAX_TAIL_CTX - 1)]); + n |= (bit << i); + } + + return n; + } + + uint32_t decode_sym(arith_data_model& dm) + { + assert(m_data_buf_size); + assert(dm.m_num_data_syms); + + uint32_t x = 0, y = m_length; + + m_length >>= DMLenShift; + + uint32_t low_idx = 0, hi_idx = dm.m_num_data_syms; + uint32_t mid_idx = hi_idx >> 1; + + do + { + uint32_t z = m_length * dm.m_cum_sym_freqs[mid_idx]; + + if (z > m_value) + { + hi_idx = mid_idx; + y = z; + } + else + { + low_idx = mid_idx; + x = z; + } + mid_idx = (low_idx + hi_idx) >> 1; + + } while (mid_idx != low_idx); + + m_value -= x; + m_length = y - x; + + if (m_length < ArithMinLen) + renorm(); + + ++dm.m_sym_freqs[low_idx]; + ++dm.m_total_sym_freq; + + if (--dm.m_num_syms_until_next_update <= 0) + dm.update(false); + + return low_idx; + } + + private: + const uint8_t* m_pData_buf; + const uint8_t* m_pData_buf_last_byte; + const uint8_t* m_pData_buf_cur; + size_t m_data_buf_size; + + uint32_t m_value, m_length; + + inline void renorm() + { + do + { + const uint32_t next_byte = (m_pData_buf_cur > m_pData_buf_last_byte) ? 0 : *m_pData_buf_cur++; + + m_value = (m_value << 8u) | next_byte; + + } while ((m_length <<= 8u) < ArithMinLen); + } + }; + + } // namespace arith +#endif // BASISD_SUPPORT_XUASTC + +#if BASISD_SUPPORT_XUASTC + namespace bc7u + { + int determine_bc7_mode(const void* pBlock); + int determine_bc7_mode_4_index_mode(const void* pBlock); + int determine_bc7_mode_4_or_5_rotation(const void* pBlock); + bool unpack_bc7_mode6(const void* pBlock_bits, color_rgba* pPixels); + bool unpack_bc7(const void* pBlock, color_rgba* pPixels); + } // namespace bc7u + + namespace bc7f + { + enum + { + // Low-level BC7 encoder configuration flags. + cPackBC7FlagUse2SubsetsRGB = 1, // use mode 1/3 for RGB blocks + cPackBC7FlagUse2SubsetsRGBA = 2, // use mode 7 for RGBA blocks + + cPackBC7FlagUse3SubsetsRGB = 4, // also use mode 0/2, cPackBC7FlagUse2SubsetsRGB MUST be enabled too + + cPackBC7FlagUseDualPlaneRGB = 8, // enable mode 4/5 usage for RGB blocks + cPackBC7FlagUseDualPlaneRGBA = 16, // enable mode 4/5 usage for RGBA blocks + + cPackBC7FlagPBitOpt = 32, // enable to disable usage of fixed p-bits on some modes; slower + cPackBC7FlagPBitOptMode6 = 64, // enable to disable usage of fixed p-bits on mode 6, alpha on fully opaque blocks may be 254 however; slower + + cPackBC7FlagUseTrivialMode6 = 128, // enable trivial fast mode 6 encoder on blocks with very low variances (highly recommended) + + cPackBC7FlagPartiallyAnalyticalRGB = 256, // partially analytical mode for RGB blocks, slower but higher quality, computes actual SSE's on complex blocks to resolve which mode to use vs. predictions + cPackBC7FlagPartiallyAnalyticalRGBA = 512, // partially analytical mode for RGBA blocks, slower but higher quality, computes actual SSE's on complex blocks to resolve which mode to use vs. predictions + + // Non-analytical is really still partially analytical on the mode pairs (0 vs. 2, 1 vs 3, 4 vs. 5). + cPackBC7FlagNonAnalyticalRGB = 1024, // very slow/brute force, totally abuses the encoder, MUST use with cPackBC7FlagPartiallyAnalyticalRGB flag + cPackBC7FlagNonAnalyticalRGBA = 2048, // very slow/brute force, totally abuses the encoder, MUST use with cPackBC7FlagPartiallyAnalyticalRGBA flag + + // Default to use first: + + // Decent analytical BC7 defaults + cPackBC7FlagDefaultFastest = cPackBC7FlagUseTrivialMode6, // very weak particularly on alpha, mode 6 only for RGB/RGBA, + + // Mode 6 with pbits for RGB, Modes 4,5,6 for alpha. + cPackBC7FlagDefaultFaster = cPackBC7FlagPBitOpt | cPackBC7FlagUseDualPlaneRGBA | cPackBC7FlagUseTrivialMode6, + + cPackBC7FlagDefaultFast = cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagUse2SubsetsRGBA | cPackBC7FlagUseDualPlaneRGBA | + cPackBC7FlagPBitOpt | cPackBC7FlagUseTrivialMode6, + + cPackBC7FlagDefault = (cPackBC7FlagUse2SubsetsRGB | cPackBC7FlagUse2SubsetsRGBA | cPackBC7FlagUse3SubsetsRGB) | + (cPackBC7FlagUseDualPlaneRGB | cPackBC7FlagUseDualPlaneRGBA) | + (cPackBC7FlagPBitOpt | cPackBC7FlagPBitOptMode6) | + cPackBC7FlagUseTrivialMode6, + + // Default partially analytical BC7 defaults (slower) + cPackBC7FlagDefaultPartiallyAnalytical = cPackBC7FlagDefault | (cPackBC7FlagPartiallyAnalyticalRGB | cPackBC7FlagPartiallyAnalyticalRGBA), + + // Default non-analytical BC7 defaults (very slow). In reality the encoder is still analytical on the mode pairs, but at the highest level is non-analytical. + cPackBC7FlagDefaultNonAnalytical = (cPackBC7FlagDefaultPartiallyAnalytical | (cPackBC7FlagNonAnalyticalRGB | cPackBC7FlagNonAnalyticalRGBA)) & ~cPackBC7FlagUseTrivialMode6 + }; + + void init(); + + void fast_pack_bc7_rgb_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + uint32_t fast_pack_bc7_rgb_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + void fast_pack_bc7_rgba_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + uint32_t fast_pack_bc7_rgba_partial_analytical(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + uint32_t fast_pack_bc7_auto_rgba(uint8_t* pBlock, const color_rgba* pPixels, uint32_t flags); + + void print_perf_stats(); + +#if 0 + // Very basic BC7 mode 6 only to ASTC. + void fast_pack_astc(void* pBlock, const color_rgba* pPixels); +#endif + + uint32_t calc_sse(const uint8_t* pBlock, const color_rgba* pPixels); + + } // namespace bc7f + + namespace etc1f + { + struct pack_etc1_state + { + uint64_t m_prev_solid_block; + //decoder_etc_block m_prev_solid_block; + + int m_prev_solid_r8; + int m_prev_solid_g8; + int m_prev_solid_b8; + + pack_etc1_state() + { + clear(); + } + + void clear() + { + m_prev_solid_r8 = -1; + m_prev_solid_g8 = -1; + m_prev_solid_b8 = -1; + } + }; + + void init(); + + void pack_etc1_solid(uint8_t* pBlock, const color_rgba& color, pack_etc1_state& state, bool init_flag = false); + + void pack_etc1(uint8_t* pBlock, const color_rgba* pPixels, pack_etc1_state& state); + + void pack_etc1_grayscale(uint8_t* pBlock, const uint8_t* pPixels, pack_etc1_state& state); + + } // namespace etc1f +#endif // BASISD_SUPPORT_XUASTC + + // Private/internal XUASTC LDR transcoding helpers + + // XUASTC LDR formats only + enum class transcoder_texture_format; + block_format xuastc_get_block_format(transcoder_texture_format tex_fmt); + +#if BASISD_SUPPORT_XUASTC + // Low-quality, but fast, PVRTC1 RGB/RGBA encoder. Power of 2 texture dimensions required. + // Note: Not yet part of our public API: this API may change! + void encode_pvrtc1( + block_format fmt, void* pDst_blocks, + const basisu::vector2D& temp_image, + uint32_t dst_num_blocks_x, uint32_t dst_num_blocks_y, bool from_alpha); + + void transcode_4x4_block( + block_format fmt, // desired output block format + uint32_t block_x, uint32_t block_y, // 4x4 block being processed + void* pDst_blocks, // base pointer to output buffer/bitmap + uint8_t* pDst_block_u8, // pointer to output block/or first pixel to write + const color32* block_pixels, // pointer to 4x4 (16) 32bpp RGBA pixels + uint32_t output_block_or_pixel_stride_in_bytes, uint32_t output_row_pitch_in_blocks_or_pixels, uint32_t output_rows_in_pixels, // output buffer dimensions + int channel0, int channel1, // channels to process, used by some block formats + bool high_quality, bool from_alpha, // Flags specific to certain block formats + uint32_t bc7f_flags, // Real-time bc7f BC7 encoder flags, see bc7f::cPackBC7FlagDefault etc. + etc1f::pack_etc1_state& etc1_pack_state, // etc1f thread local state + int has_alpha = -1); // has_alpha = -1 unknown, 0=definitely no (a all 255's), 1=potentially yes +#endif // BASISD_SUPPORT_XUASTC + + struct bc7_mode_5 + { + union + { + struct + { + uint64_t m_mode : 6; + uint64_t m_rot : 2; + + uint64_t m_r0 : 7; + uint64_t m_r1 : 7; + uint64_t m_g0 : 7; + uint64_t m_g1 : 7; + uint64_t m_b0 : 7; + uint64_t m_b1 : 7; + uint64_t m_a0 : 8; + uint64_t m_a1_0 : 6; + + } m_lo; + + uint64_t m_lo_bits; + }; + + union + { + struct + { + uint64_t m_a1_1 : 2; + + // bit 2 + uint64_t m_c00 : 1; + uint64_t m_c10 : 2; + uint64_t m_c20 : 2; + uint64_t m_c30 : 2; + + uint64_t m_c01 : 2; + uint64_t m_c11 : 2; + uint64_t m_c21 : 2; + uint64_t m_c31 : 2; + + uint64_t m_c02 : 2; + uint64_t m_c12 : 2; + uint64_t m_c22 : 2; + uint64_t m_c32 : 2; + + uint64_t m_c03 : 2; + uint64_t m_c13 : 2; + uint64_t m_c23 : 2; + uint64_t m_c33 : 2; + + // bit 33 + uint64_t m_a00 : 1; + uint64_t m_a10 : 2; + uint64_t m_a20 : 2; + uint64_t m_a30 : 2; + + uint64_t m_a01 : 2; + uint64_t m_a11 : 2; + uint64_t m_a21 : 2; + uint64_t m_a31 : 2; + + uint64_t m_a02 : 2; + uint64_t m_a12 : 2; + uint64_t m_a22 : 2; + uint64_t m_a32 : 2; + + uint64_t m_a03 : 2; + uint64_t m_a13 : 2; + uint64_t m_a23 : 2; + uint64_t m_a33 : 2; + + } m_hi; + + uint64_t m_hi_bits; + }; + }; + } // namespace basist + + + diff --git a/webgl/encoder/CMakeLists.txt b/webgl/encoder/CMakeLists.txt index 48ada14..e8e5adb 100644 --- a/webgl/encoder/CMakeLists.txt +++ b/webgl/encoder/CMakeLists.txt @@ -1,94 +1,136 @@ cmake_minimum_required(VERSION 3.5) - project(basisu_encoder_js) -# The encoder always supports generating KTX2 files, but Zstandard support is optional. If it's disabled, KTX2 UASTC files will always be uncompressed. -# If you know you'll never be encoding UASTC+Zstd KTX2 files you can set KTX2_ZSTANDARD to 0 to reduce the size of the compiled encoder. -option(KTX2_ZSTANDARD "KTX2_ZSTANDARD" TRUE) - +# Toggle Zstd support for KTX2 (ON by default). +option(KTX2_ZSTANDARD "Enable KTX2 Zstandard support" TRUE) message("KTX2_ZSTANDARD=${KTX2_ZSTANDARD}") -if (EMSCRIPTEN) +# Only for Emscripten builds. +if(EMSCRIPTEN) set(CMAKE_CXX_STANDARD 17) + # ---------- Pick config once (single-config generators) ---------- + # Supports: Release (default), Debug, SAN + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release") + endif() + string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_MODE) + + # Per-config compile/link flags + set(CONFIG_CFLAGS "") + set(CONFIG_DEFS "") + set(CONFIG_LINK "") + + if(BUILD_MODE STREQUAL "RELEASE") + set(CONFIG_CFLAGS "-O3") + set(CONFIG_DEFS "NDEBUG") + set(CONFIG_LINK "-O3 -s ASSERTIONS=0") + elseif(BUILD_MODE STREQUAL "DEBUG") + set(CONFIG_CFLAGS "-g -O0") + set(CONFIG_DEFS "DEBUG") + set(CONFIG_LINK "-g -s ASSERTIONS=2") + elseif(BUILD_MODE STREQUAL "SAN") + set(CONFIG_CFLAGS "-g -O1 -fsanitize=undefined -fsanitize=address") + set(CONFIG_DEFS "DEBUG") + set(CONFIG_LINK "-g -s ASSERTIONS=2 -fsanitize=undefined -fsanitize=address") + else() + message(WARNING "Unknown CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}', defaulting to Release-like flags.") + set(CONFIG_CFLAGS "-O3") + set(CONFIG_DEFS "NDEBUG") + set(CONFIG_LINK "-O3 -s ASSERTIONS=0") + endif() + + # ---------- Sources (shared) ---------- set(SRC_LIST ../transcoder/basis_wrappers.cpp ../../transcoder/basisu_transcoder.cpp - ../../encoder/basisu_backend.cpp - ../../encoder/basisu_basis_file.cpp - ../../encoder/basisu_comp.cpp - ../../encoder/basisu_enc.cpp - ../../encoder/basisu_etc.cpp - ../../encoder/basisu_frontend.cpp - ../../encoder/basisu_gpu_texture.cpp - ../../encoder/basisu_pvrtc1_4.cpp - ../../encoder/basisu_resampler.cpp - ../../encoder/basisu_resample_filters.cpp - ../../encoder/basisu_ssim.cpp - ../../encoder/basisu_uastc_enc.cpp - ../../encoder/basisu_bc7enc.cpp - ../../encoder/basisu_kernels_sse.cpp - ../../encoder/basisu_opencl.cpp - ../../encoder/pvpngreader.cpp - ../../encoder/jpgd.cpp - ../../encoder/3rdparty/android_astc_decomp.cpp - ../../encoder/basisu_uastc_hdr_4x4_enc.cpp - ../../encoder/basisu_astc_hdr_6x6_enc.cpp - ../../encoder/basisu_astc_hdr_common.cpp - ../../encoder/3rdparty/tinyexr.cpp + ../../encoder/basisu_backend.cpp + ../../encoder/basisu_basis_file.cpp + ../../encoder/basisu_comp.cpp + ../../encoder/basisu_enc.cpp + ../../encoder/basisu_etc.cpp + ../../encoder/basisu_frontend.cpp + ../../encoder/basisu_gpu_texture.cpp + ../../encoder/basisu_pvrtc1_4.cpp + ../../encoder/basisu_resampler.cpp + ../../encoder/basisu_resample_filters.cpp + ../../encoder/basisu_ssim.cpp + ../../encoder/basisu_uastc_enc.cpp + ../../encoder/basisu_bc7enc.cpp + ../../encoder/basisu_kernels_sse.cpp + ../../encoder/basisu_opencl.cpp + ../../encoder/pvpngreader.cpp + ../../encoder/jpgd.cpp + ../../encoder/3rdparty/android_astc_decomp.cpp + ../../encoder/basisu_uastc_hdr_4x4_enc.cpp + ../../encoder/basisu_astc_hdr_6x6_enc.cpp + ../../encoder/basisu_astc_hdr_common.cpp + ../../encoder/basisu_astc_ldr_common.cpp + ../../encoder/basisu_astc_ldr_encode.cpp + ../../encoder/3rdparty/tinyexr.cpp ) - - if (KTX2_ZSTANDARD) - set(SRC_LIST ${SRC_LIST} - ../../zstd/zstd.c - ) - set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=1) + if(KTX2_ZSTANDARD) + list(APPEND SRC_LIST ../../zstd/zstd.c) + set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=1) else() - set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=0) + set(ZSTD_DEFINITION BASISD_SUPPORT_KTX2_ZSTD=0) endif() - # No threading version - add_executable(basis_encoder.js ${SRC_LIST}) + # Common preprocessor defines (same as your original) + set(COMMON_DEFS + BASISD_SUPPORT_UASTC=1 + BASISD_SUPPORT_BC7=1 + BASISD_SUPPORT_ATC=0 + BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 + BASISD_SUPPORT_PVRTC2=0 + BASISD_SUPPORT_FXT1=0 + BASISD_SUPPORT_ETC2_EAC_RG11=0 + BASISU_SUPPORT_ENCODING=1 + BASISU_SUPPORT_SSE=0 + BASISD_SUPPORT_XUASTC=1 + ${ZSTD_DEFINITION} + ) - #target_compile_definitions(basis_encoder.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -O3) + # Base link flags + set(LINK_BASE "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -s STACK_SIZE=2097152 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") + set(LINK_THREADS "-s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker") + set(LINK_WASM64 "-s MEMORY64=1 -sWASM_BIGINT=1 --profiling-funcs") - #target_compile_definitions(basis_encoder.js PRIVATE DEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -g -O1 -fsanitize=undefined -fsanitize=address) + # Helper to avoid repetition + function(add_encoder target out_name use_threads use_wasm64) + add_executable(${target} ${SRC_LIST}) + set_target_properties(${target} PROPERTIES OUTPUT_NAME "${out_name}" SUFFIX ".js") + target_include_directories(${target} PRIVATE ../../transcoder) - # debug options - #target_compile_definitions(basis_encoder.js PRIVATE DEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - #target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -g -O0) + # Compile defs and options + target_compile_definitions(${target} PRIVATE ${COMMON_DEFS} ${CONFIG_DEFS}) + target_compile_options(${target} PRIVATE -fno-strict-aliasing ${CONFIG_CFLAGS}) - # release options - target_compile_definitions(basis_encoder.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} ) - target_compile_options(basis_encoder.js PRIVATE -fno-strict-aliasing -O3) + if(${use_threads}) + target_compile_options(${target} PRIVATE -matomics -mbulk-memory) + target_compile_definitions(${target} PRIVATE WASM_THREADS_ENABLED=1) + endif() + + if(${use_wasm64}) + target_compile_options(${target} PRIVATE -s MEMORY64=1) + endif() - target_include_directories(basis_encoder.js PRIVATE ../../transcoder) - - set_target_properties(basis_encoder.js PROPERTIES - OUTPUT_NAME "basis_encoder" - SUFFIX ".js" - - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -s ALLOW_MEMORY_GROWTH=1 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -fsanitize=undefined -fsanitize=address -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=2 -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - LINK_FLAGS "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=0 -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - - add_executable(basis_encoder_threads.js ${SRC_LIST}) - - # Threaded version - target_compile_definitions(basis_encoder_threads.js PRIVATE NDEBUG BASISD_SUPPORT_UASTC=1 BASISD_SUPPORT_BC7=1 BASISD_SUPPORT_ATC=0 BASISD_SUPPORT_ASTC_HIGHER_OPAQUE_QUALITY=0 BASISD_SUPPORT_PVRTC2=0 BASISD_SUPPORT_FXT1=0 BASISD_SUPPORT_ETC2_EAC_RG11=0 BASISU_SUPPORT_ENCODING=1 BASISU_SUPPORT_SSE=0 ${ZSTD_DEFINITION} WASM_THREADS_ENABLED=1 ) - target_include_directories(basis_encoder_threads.js PRIVATE ../../transcoder) - - target_compile_options(basis_encoder_threads.js PRIVATE -fno-strict-aliasing -O3 -matomics -mbulk-memory) - - set_target_properties(basis_encoder_threads.js PROPERTIES - OUTPUT_NAME "basis_encoder_threads" - SUFFIX ".js" - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -s ALLOW_MEMORY_GROWTH=1 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -fsanitize=undefined -fsanitize=address -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - #LINK_FLAGS "--bind -s INITIAL_MEMORY=536870912 -g -s STACK_SIZE=262144 -s ALLOW_MEMORY_GROWTH=1 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=2 -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker -s EXPORTED_RUNTIME_METHODS=['HEAP8']") - LINK_FLAGS "--bind -s ALLOW_MEMORY_GROWTH=1 -s INITIAL_MEMORY=536870912 -O3 -s STACK_SIZE=262144 -s MODULARIZE=1 -s EXPORT_NAME=BASIS -s ASSERTIONS=0 -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=18 -s ENVIRONMENT=web,worker -s EXPORTED_RUNTIME_METHODS=['HEAP8']") + # Link flags (no generator expressions) + set(_lf "${LINK_BASE} ${CONFIG_LINK}") + + if(${use_threads}) + set(_lf "${_lf} ${LINK_THREADS}") + endif() + + if(${use_wasm64}) + set(_lf "${_lf} ${LINK_WASM64} -s INITIAL_MEMORY=1073741824 -s MAXIMUM_MEMORY=12884901888") + endif() + + set_target_properties(${target} PROPERTIES LINK_FLAGS "${_lf}") + endfunction() + # The three outputs (names unchanged) + add_encoder(basis_encoder.js "basis_encoder" OFF OFF) # wasm32 + add_encoder(basis_encoder_threads.js "basis_encoder_threads" ON OFF) # wasm32 + threads + add_encoder(basis_encoder_threads_wasm64.js "basis_encoder_threads_wasm64" ON ON ) # wasm64 + threads endif() diff --git a/webgl/index.html b/webgl/index.html index d469dcd..f998dfb 100644 --- a/webgl/index.html +++ b/webgl/index.html @@ -6,12 +6,12 @@

Basis Universal LDR/HDR WebGL demos

- Example web pages using the transcoder (compiled to WASM) to render - .KTX2/.basis textures in WebGL, and the encoder to encode .KTX2 files. + Example web pages using the transcoder library, compiled to WASM, to render + .KTX2/.basis textures in WebGL, and the compressor library to encode .KTX2 files.

diff --git a/webgl/ktx2_encode_test/index.html b/webgl/ktx2_encode_test/index.html index 8f98e92..4b3956f 100644 --- a/webgl/ktx2_encode_test/index.html +++ b/webgl/ktx2_encode_test/index.html @@ -1,944 +1,1338 @@ - + + + + if (bc6hSupported) + { + if (bc6hDisabled) + addButton(''); + else + addButton(''); + } + + if (rgbaHalfSupported) + { + if (rgbaHalfDisabled) + addButton(''); + else + addButton(''); + } + + addButton(''); + addButton(''); + } + + function updateSupportedFormats() + { + let supportedFormats = []; + + // Collect all supported formats + if (astcSupported && !astcDisabled) + supportedFormats.push('ASTC LDR'); + + if (etcSupported && !etcDisabled) + supportedFormats.push('ETC1'); + + if (dxtSupported && !dxtDisabled) + supportedFormats.push('BC1-5'); + + if (pvrtcSupported && !pvrtcDisabled) + supportedFormats.push('PVRTC'); + + if (bc7Supported && !bc7Disabled) + supportedFormats.push('BC7'); + + if (astcHDRSupported && !astcHDRDisabled) + supportedFormats.push('ASTC HDR'); + + if (bc6hSupported && !bc6hDisabled) + supportedFormats.push('BC6H'); + + if (rgbaHalfSupported && !rgbaHalfDisabled) + supportedFormats.push('RGBA_HALF'); + + // Prepare the display string + let supportedString = 'Supported WebGL formats: '; + + if (supportedFormats.length > 0) + { + for (let i = 0; i < supportedFormats.length; i++) + { + supportedString += supportedFormats[i]; + // Add a comma after each format except the last + if (i < supportedFormats.length - 1) + { + supportedString += ', '; + } + + // Start a new line after every 4 formats + if ((i + 1) % 4 === 0) + { + supportedString += '
'; + } + } + } + else + { + supportedString = 'No WebGL formats detected/enabled; using uncompressed fallback formats.'; + } + + // Update the HTML element + elem('supported-formats').innerHTML = supportedString; + } + + function checkForGPUFormatSupport(gl) + { + astcSupported = !!gl.getExtension('WEBGL_compressed_texture_astc'); + etcSupported = !!gl.getExtension('WEBGL_compressed_texture_etc1'); + dxtSupported = !!gl.getExtension('WEBGL_compressed_texture_s3tc'); + pvrtcSupported = !!(gl.getExtension('WEBGL_compressed_texture_pvrtc')) || !!(gl.getExtension('WEBKIT_WEBGL_compressed_texture_pvrtc')); + bc7Supported = !!gl.getExtension('EXT_texture_compression_bptc'); + + // Check for BC6H support + { + var ext = gl.getExtension('EXT_texture_compression_bptc'); + if (ext) + { + bc6hSupported = !!ext.COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT_EXT; + } + } + + // Check for ASTC HDR support + { + var ext = gl.getExtension('WEBGL_compressed_texture_astc'); + + if (ext) + { + var supportedProfiles = ext.getSupportedProfiles(); + + var hdrProfiles = supportedProfiles.filter(profile => profile.includes('hdr')); + + if (hdrProfiles.length > 0) + { + astcHDRSupported = true; + } + } + } + + // Check for half-float texture support. + { + var ext = gl.getExtension('OES_texture_half_float'); + if (ext) + { + rgbaHalfSupported = true; + halfFloatWebGLFormat = ext.HALF_FLOAT_OES; + } + } + + // HACK HACK - for testing uncompressed fallbacks. + //astcSupported = false; + //etcSupported = false; + //dxtSupported = false; + //bc7Supported = false; + //pvrtcSupported = false; + //bc6hSupported = false; + //astcHDRSupported = false; + //rgbaHalfSupported = false; + + console.log('astcSupported: ' + astcSupported); + console.log('etcSupported: ' + etcSupported); + console.log('dxtSupported: ' + dxtSupported); + console.log('bc7Supported: ' + bc7Supported); + console.log('pvrtcSupported: ' + pvrtcSupported); + console.log('bc6hSupported: ' + bc6hSupported); + console.log('astcHDRSupported: ' + astcHDRSupported); + console.log('rgbaHalfSupported: ' + rgbaHalfSupported); + + updateSupportedFormats(); + updateDisableButtons(); + } + + function validateNitMultiplier(input) + { + const warning = document.getElementById('ldr-to-hdr-warning'); + const value = parseFloat(input.value); + + if (isNaN(value) || value < 0.1 || value > 1000) + { + // Show warning if value is invalid + warning.style.display = 'inline'; + } + else + { + // Hide warning if value is valid + warning.style.display = 'none'; + } + } + + function getNitMultiplier() + { + const input = document.getElementById('ldr-to-hdr-multiplier'); + const value = parseFloat(input.value); + + if (!isNaN(value) && value >= 0.1 && value <= 1000) + { + return value; + } + else + { + log('Invalid LDR to HDR Nit Multiplier value. Defaulting to 100.0'); + return 100.0; // Default value if invalid + } + } + + function getSelectedBasisTexFormat() + { + const selectElem = document.getElementById('basis-tex-format'); + const selectedValue = parseInt(selectElem.value, 10); + + return selectedValue; + } + + function getXUASTCLDRSyntax() + { + const selectElem = document.getElementById('xuastc_ldr_syntax'); + const selectedValue = parseInt(selectElem.value, 10); + + return selectedValue; + } + + function getASTC6x6RDOLambda() + { + const input = document.getElementById('astc6x6-rdo-lambda'); + const value = parseFloat(input.value); + + if (!isNaN(value) && value >= 0.0 && value <= 1000000) + { + return value; + } + else + { + log('Invalid lambda value. Defaulting to 0.0'); + return 0.0; // Default value if invalid + } + } + + function showBusyModal() + { + //console.log("Showing modal"); + document.getElementById('busy-modal').style.display = 'flex'; + } + + function hideBusyModal() + { + //console.log("Hiding modal"); + document.getElementById('busy-modal').style.display = 'none'; + } + + function setDropdownValue(selectId, value) + { + const selectElement = document.getElementById(selectId); + if (!selectElement) + return; + + selectElement.value = value; + selectElement.dispatchEvent(new Event('change')); + } + + function globalInit() + { + elem('SRGB').checked = true; + + var gl = elem('canvas').getContext('webgl'); + checkForGPUFormatSupport(gl); + + window.renderer = new Renderer(gl); + + elem('file').addEventListener('keydown', function (e) + { + if (e.keyCode == 13) { + runLoadFile(); + } + }, false); + + elem('imagefile').addEventListener('keydown', function (e) + { + if (e.keyCode == 13) { + runEncodeImageFile(); + } + }, false); + + { + let etc1SLevelSlider = document.getElementById('etc1s-comp-level-slider'); + let etc1sLevelSliderValueDisplay = document.getElementById('etc1s-comp-level-slider-value'); + etc1SLevelSlider.oninput = function() + { + etc1sLevelSliderValueDisplay.textContent = this.value; + } + } + + { + let uastcHDRSlider = document.getElementById('uastc-hdr-quality-slider'); + let qualityHDRValueDisplay = document.getElementById('uastc-hdr-quality-value'); + uastcHDRSlider.oninput = function() + { + qualityHDRValueDisplay.textContent = this.value; + } + } + + { + let astcHDR6x6Slider = document.getElementById('astc-hdr6x6-comp-level-slider'); + let compLevelValueDisplay = document.getElementById('astc-hdr6x6-comp-level-value'); + astcHDR6x6Slider.oninput = function() + { + compLevelValueDisplay.textContent = this.value; + } + } + + { + let uastcLDRSlider = document.getElementById('uastc-ldr-quality-slider'); + let qualityLDRValueDisplay = document.getElementById('uastc-ldr-quality-value'); + uastcLDRSlider.oninput = function() + { + qualityLDRValueDisplay.textContent = this.value; + } + } + + { + let rdoSlider = document.getElementById('rdo-quality-slider'); + let rdoValueDisplay = document.getElementById('rdo-quality-value'); + rdoSlider.oninput = function() + { + rdoValueDisplay.textContent = parseFloat(this.value).toFixed(1); + } + } + + { + let etc1SQualitySlider = document.getElementById('EncodeQuality'); + let etc1SQualitySliderValue = document.getElementById('encode-quality-value'); + etc1SQualitySlider.oninput = function() + { + etc1SQualitySliderValue.textContent = parseFloat(this.value).toFixed(0); + } + } + + { + let xuastcLDREffortSlider = document.getElementById('xuastc_ldr_effort_level_slider'); + let xuastcLDREffortValue = document.getElementById('xuastc_ldr_effort_level_value'); + xuastcLDREffortSlider.oninput = function() + { + xuastcLDREffortValue.textContent = this.value; + } + } + + { + let xuastcLDRDCTSlider = document.getElementById('xuastc_ldr_dct_quality_slider'); + let xuastcLDRDCTValue = document.getElementById('xuastc_ldr_dct_quality_value'); + xuastcLDRDCTSlider.oninput = function() + { + xuastcLDRDCTValue.textContent = this.value; + } + } + + { + document.getElementById("unified-effort-slider").oninput = function() + { + document.getElementById("unified-effort-value").textContent = this.value; + } + + document.getElementById("unified-quality-slider").oninput = function() + { + document.getElementById("unified-quality-value").textContent = this.value; + } + } + + runLoadFile(); + } + + function updateErrorLine(message) + { + const errorLine = document.getElementById('error-line'); + errorLine.textContent = message; + errorLine.style.color = message.trim() ? 'red' : ''; + } + + + + + + -
-
- Basis Universal Multithreaded .KTX2 Supercompressed GPU Texture Encoding/Transcoding Testbed v0.67 -
-
This demo uses the Basis Universal C++ transcoder (compiled to WebAssembly using Emscripten) to transcode a .ktx2 file to FORMAT -
It also supports encoding .PNG, .EXR or .HDR files to LDR or HDR .KTX2 files. -
Thanks to Evan Parker for providing webgl-texture-utils and this test bed. Go back. -
Enable your browser debug console (F12 on Chrome/Firefox) to see debug output. -
Note: The largest image resolution that can be compressed in the browser with -
this library is limited to around 6 megapixels due to 32-bit WASM memory -
constraints. Larger images risk running out of memory. +
+
-

-
- Supported WebGL formats: -
+
+
+ Basis Universal .KTX2 Supercompressed GPU Texture Encoding/Transcoding Testbed v2.01 +
-
-
- Test -
+
This simple demo uses the Basis Universal C++ transcoder (compiled to WebAssembly using Emscripten) to transcode a .ktx2 file to: +
FORMAT +
+
The viewer is implemented in WebGL and renders a single textured quad. It also supports encoding .PNG, .JPG, .EXR or .HDR files to LDR or HDR .KTX2 files. + Thanks to Evan Parker for providing webgl-texture-utils and this test bed. Go back. + Notes: Enable your browser debug console (F12 on Chrome/Firefox) to see debug output. + The largest image resolution that can be compressed in the browser with + this library is limited to either 12 megapixels or 4 megapixels (depending on format and WASM64/WASM32) to avoid running out of WASM memory. -
- -
+

+
+ Supported WebGL formats: +
-
-
- Use Multithreading (if available) +
+
+ Test +
-
Additional Worker Threads (Max 18): - -

+
+ +
-
- .ktx2 file: - - -
+
+
+
+ Use Multithreading (if available) -
- .png/.jpg/.exr/.hdr file: - - -
-
- +

- - - +
+ .ktx2 file: + + +
- - +
+ .png/.jpg/.exr/.hdr file: + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + -
+ + + -
-

Drag and drop a PNG/JPG/EXR/HDR file here, or click to select a file.

+ + + + + + +
+ +
+

Drag and drop a .KTX2 or image file here, or click to select a file.

-
-
+
+
- + -
+
+ + KTX2 Texture Format to Encode: + + + + + + + + + + + + + + + -
-
+ + +
+
+ + Primary compression quality/effort options: - Higher quality ASTC 6x6 HDR->BC6H transcoding: - +
+ Use unified quality/effort options (overrides below low-level options): + -

- - - +
+ + + 2 -
-
- Disabled +

-
+ + + 80 - - - 1 +
+
+ + Transcoder Options (Decode Flags): -
- -
+
+ ETC1S: No BC7 Chroma Artifact Filtering (faster transcoding): + -
-
+
+ XUASTC/ASTC LDR: Disable deblocking filtering (faster): + - KTX2 Texture Format to Encode: - -
+
+ XUASTC/ASTC LDR: Stronger deblocking filtering: + -
+
+ XUASTC/ASTC LDR: Use deblocking on all block sizes (slower): + - ETC1S LDR Options: -
- ETC1S Quality: - - 255 +
+ XUASTC LDR 4x4/6x6/8x6: No direct BC7 transcoding (slower/higher quality): + -
- - - 2 +
-
+ Prefer higher quality transcoding when supported (slower): + - UASTC LDR 4x4 Options: -
- - - 1 +
+
+ + Display/Visualization Options: +
+ + + + -
+
- UASTC LDR RDO: - +
+ Disabled - - - 1.0 +
-
+ + + 1 + +
- UASTC HDR 4x4 Options: +
-
- - - 0 -
+ Low-level ETC1S LDR Options: +
+ ETC1S Quality: + + 255 -
+
+ + + 1 - ASTC HDR 6x6 Options: -
+
- - - 0 -
+ Low-level UASTC LDR 4x4 Options: - RDO Quality (Lambda, 0-50k, try 0-5k, higher=smaller): - -
+
+ UASTC LDR RDO: + - REC 2020 Colorspace: - + + + 1.0 -
+
+ + + 1 - LDR->HDR Upconversion Options: -
- Convert LDR images to linear light: - +
-
- LDR to HDR Upconversion Nit Multiplier: - - -
+
-
+ Low-level UASTC HDR 4x4 Options: - Other Options: +
+ + + 0 +
-
- Use sRGB/perceptual metrics: - -
- Generate mipmaps: - +
-
-
- Debug Output (See Dev Console): - - Compute Stats: - + Low-level ASTC/UASTC HDR 6x6 Options: +
-
+ + + 0 +
- Log Output: + RDO Quality (Lambda, 0-50k, try 0-5k, higher=smaller): + +
-
+ REC 2020 Colorspace: + - +
+ + LDR->HDR Upconversion Options: +
+ Convert LDR images to linear light: + + +
+ LDR to HDR Upconversion Nit Multiplier: + + +
+ +
+ Low-level XUASTC/ASTC LDR 4x4-12x12 Options: +
+ + XUASTC LDR Syntax: + + +
+ + + + 3 +
+ + + + 80 + +
+ Bounded/windowed RDO lossy supercompression: + + +
+ No RGB dual plane (lower quality, faster encoding/BC7 transcoding): + + +
+ No 2-3 subset usage (lower quality, faster encoding/BC7 transcoding): + + +

+ ASTC/XUASTC LDR Bounded/Windowed RDO Params: + +
+ Opaque: +
+ +
+ +
+ + +
+ +
Alpha:
+ +
+ +
+ +
+ +
+ Other Options: + +
+ Use sRGB/perceptual metrics: + + +
+ Generate mipmap levels: + + +
+ Y flip source image: + + +
+
+ Debug Output (See Dev Console): + + Compute Stats (slower encoding): + + +
+ + Log Output: +
+
+ + + +
+ +
+ +
+ +
+ + + + - - - - - diff --git a/webgl/ktx2_encode_test/renderer.js b/webgl/ktx2_encode_test/renderer.js index 8de7e58..cfd1b00 100644 --- a/webgl/ktx2_encode_test/renderer.js +++ b/webgl/ktx2_encode_test/renderer.js @@ -10,7 +10,7 @@ var Renderer = function (gl) { * @private */ this.gl_ = gl; - + /** * The WebGLProgram. * @type {WebGLProgram} @@ -53,11 +53,19 @@ var Renderer = function (gl) { */ this.quadVertexBuffer_ = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, this.quadVertexBuffer_); + var vertices = new Float32Array( [-1.0, -1.0, 0.0, 1.0, +1.0, -1.0, 1.0, 1.0, -1.0, +1.0, 0.0, 0.0, 1.0, +1.0, 1.0, 0.0]); + +// var vertices = new Float32Array( +// [-1.0, -1.0, 0.0, .5, +// +1.0, -1.0, .5, .5, +// -1.0, +1.0, 0.0, 0.0, +// 1.0, +1.0, .5, 0.0]); + gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW); @@ -90,7 +98,7 @@ var Renderer = function (gl) { Renderer.prototype.finishInit = function () { - this.draw(); + //this.draw(); }; @@ -156,12 +164,12 @@ Renderer.prototype.createHalfRGBATexture = function (data, width, height, format return tex; }; -// WebGL requires each row of rgb565Data to be aligned on a 4-byte boundary. +// WebGL requires each row of rgb565Data to be aligned on a 4-byte boundary. Renderer.prototype.createRgb565Texture = function (rgb565Data, width, height) { var gl = this.gl_; var tex = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, tex); - + gl.texImage2D( gl.TEXTURE_2D, 0, @@ -203,7 +211,7 @@ Renderer.prototype.createRgbaTexture = function (rgbaData, width, height) { }; -Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, linearToSRGBFlag) { +Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, linearToSRGBFlag, useLinearFiltering) { var gl = this.gl_; // draw scene gl.clearColor(0, 0, 0, 1); @@ -213,6 +221,11 @@ Renderer.prototype.drawTexture = function (texture, width, height, mode, scale, gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); + + // Point vs. bilinear sampling (no mipmaps involved here) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, useLinearFiltering ? gl.LINEAR : gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, useLinearFiltering ? gl.LINEAR : gl.NEAREST); + gl.uniform1i(this.uniformLocations_.texSampler, 0); var x = 0.0; @@ -250,7 +263,7 @@ Renderer.prototype.compileShader_ = function (shaderSource, type) { var shader = gl.createShader(type); gl.shaderSource(shader, shaderSource); gl.compileShader(shader); - + // Check for errors const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS); if (!compiled) { @@ -259,7 +272,7 @@ Renderer.prototype.compileShader_ = function (shaderSource, type) { gl.deleteShader(shader); // Cleanup shader object throw new Error('Shader compilation failed'); } - + return shader; }; @@ -310,3 +323,4 @@ Renderer.fragmentShaderSource_ = [ ' gl_FragColor = c;', '}' ].join('\n'); + diff --git a/webgl/transcoder/basis_wrappers.cpp b/webgl/transcoder/basis_wrappers.cpp index 091dd79..04349d8 100644 --- a/webgl/transcoder/basis_wrappers.cpp +++ b/webgl/transcoder/basis_wrappers.cpp @@ -11,11 +11,11 @@ // getFileDesc(), getImageDesc(), getImageLevelDesc(): These functions return low-level information about where compressed data is located for each image in a .basis file. // This is useful for when you want to extract the compressed data and embed it into your own file formats, for container independent transcoding. // -// 2. Encoding (optional): See class basis_encoder. Encodes LDR .PNG or 32bpp images, or HDR half-float/float or .EXR/.HDR images to .basis/.ktx2 files in memory. +// 2. Encoding (optional): See class basis_encoder. Encodes LDR .PNG or 32bpp images, or HDR half-float/float or .EXR/.HDR images to .basis/.ktx2 files in memory. // Must compile with BASISU_SUPPORT_ENCODING=1. // Requires basisu_transcoder.cpp as well as all the .cpp files in the "encoder" directory. Results in a larger WebAssembly executable. // -// 3. Low level transcoding/container independent transcoding: See class lowlevel_etc1s_image_transcoder or function transcodeUASTCImage(). +// 3. Low level transcoding/container independent transcoding: See class lowlevel_etc1s_image_transcoder or function transcodeUASTCImage(). // For transcoding raw compressed ETC1S/UASTC LDR/UASTC HDR texture data from non-.basis files (say from KTX2) to GPU texture data. // // 4. Helpers, transcoder texture format information: See functions getBytesPerBlockOrPixel(), formatHasAlpha(), etc. @@ -27,10 +27,20 @@ // Enable debug printf()'s in this module. #ifndef BASISU_DEBUG_PRINTF +// DO NOT CHECK IN #define BASISU_DEBUG_PRINTF 0 #endif -#define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (6291456) +// This check can be removed, but you risk crashing on larger images in 32-bit WASM. Also, ETC1S/UASTC LDR 4x4 encoding uses way less memory than UASTC HDR 6x6 encoding, so you could boost this in those cases. +// 32-bit WASM limitation (TODO: remove for 64-bit), to prevent OOM crashes during HDR encoding in particular. +// TODO: Even WASM64 in Chrome has limits which seem too low for us. For now, just impose this limit. +#ifdef __wasm64__ + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (1024*1024*12) + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT (1024*1024*12) +#else + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS (1024*1024*4) + #define BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT (1024*1024*12) +#endif #include "basisu_transcoder.h" #include @@ -60,7 +70,14 @@ void basis_init() std::lock_guard lock(s_init_mutex); #if BASISU_DEBUG_PRINTF - printf("basis_init()\n"); + printf("basis_init() " BASISD_VERSION_STRING " "); +#ifdef __wasm64__ + printf("WASM64 "); +#endif +#ifdef WASM_THREADS_ENABLED + printf("PTHREADS"); +#endif + printf("\n"); #endif if (g_basis_initialized_flag) @@ -75,17 +92,24 @@ void basis_init() g_basis_initialized_flag = true; } -static void copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) +#if 0 +// Old copy methods, used in previous builds for plain WASM (not WASM64). + +// false if resize() fails +static bool copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) { unsigned int length = srcBuffer["length"].as(); - dstVec.resize(length); + if (!dstVec.try_resize(length)) + return false; emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = srcBuffer["constructor"].new_(memory, reinterpret_cast(dstVec.data()), length); // Copy the bytes from the Javascript buffer. memoryView.call("set", srcBuffer); + + return true; } static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vector& srcVec) @@ -118,6 +142,36 @@ static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vec return true; } +#else +// New methods, compatible with WASM64. +static bool copy_from_jsbuffer(const emscripten::val& srcBuffer, basisu::vector& dstVec) +{ + const size_t length = srcBuffer["length"].as(); + if (!dstVec.try_resize(length)) + return false; + + // View over dstVec in WASM memory; copy from JS buffer into it. + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(length, dstVec.data())); + dstView.call("set", srcBuffer); + return true; +} + +// WASM -> JS +static bool copy_to_jsbuffer(const emscripten::val& dstBuffer, const basisu::vector& srcVec) +{ + if (srcVec.empty()) + return false; + + const size_t dstLen = dstBuffer["byteLength"].as(); + if (srcVec.size() > dstLen) + return false; + + // View over srcVec; copy into provided JS TypedArray. + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(srcVec.size(), const_cast(srcVec.data()))); + dstBuffer.call("set", srcView); + return true; +} +#endif const uint32_t BASIS_MAGIC = 0xD4ADBEA1; const uint32_t KTX2_MAGIC = 0xD4ADBEF2; @@ -133,7 +187,7 @@ struct basis_file_desc uint32_t m_userdata0; uint32_t m_userdata1; - // Type of texture (cETC1S, cUASTC4x4, cUASTC_HDR_4x4, etc.) + // Type of texture (cETC1S, cUASTC_LDR_4x4, cUASTC_HDR_4x4, etc.) uint32_t m_tex_format; // basis_tex_format bool m_y_flipped; @@ -190,10 +244,7 @@ struct basis_file basisu::vector m_file; basis_file(const emscripten::val& jsBuffer) - : m_file([&]() { - size_t byteLength = jsBuffer["byteLength"].as(); - return basisu::vector(byteLength); - }()) + : m_file(jsBuffer["byteLength"].as()) { if (!g_basis_initialized_flag) { @@ -204,11 +255,27 @@ struct basis_file return; } +#if 0 unsigned int length = jsBuffer["length"].as(); emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = jsBuffer["constructor"].new_(memory, reinterpret_cast(m_file.data()), length); memoryView.call("set", jsBuffer); +#else + const size_t n = jsBuffer["byteLength"].as(); + if (!n) + { +#if BASISU_DEBUG_PRINTF + printf("basis_file::basis_file: zero size file\n"); +#endif + m_file.clear(); + return; + } + + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(n, m_file.data())); + dstView.call("set", jsBuffer); +#endif + if (!m_transcoder.validate_header(m_file.data(), m_file.size())) { #if BASISU_DEBUG_PRINTF @@ -291,7 +358,7 @@ struct basis_file return orig_height; } - // Returns a basis_tex_format (cETC1S, cUASTC, cUASTC_HDR_4x4, etc.) + // Returns a basis_tex_format (cETC1S, cUASTC_LDR_4x4, cUASTC_HDR_4x4, etc. - see basiu_file_headers.h) uint32_t getBasisTexFormat() { assert(m_magic == BASIS_MAGIC); @@ -302,7 +369,7 @@ struct basis_file return (uint32_t)fmt; } - // Currently 4 or 6 + // Returns 4-12 uint32_t getBlockWidth() const { assert(m_magic == BASIS_MAGIC); @@ -313,7 +380,7 @@ struct basis_file return basis_tex_format_get_block_width(fmt); } - // Currently 4 or 6 + // Returns 4-12 uint32_t getBlockHeight() { assert(m_magic == BASIS_MAGIC); @@ -423,12 +490,13 @@ struct basis_file return result; } + // format is transcoder_texture_format uint32_t getImageTranscodedSizeInBytes(uint32_t image_index, uint32_t level_index, uint32_t format) { assert(m_magic == BASIS_MAGIC); if (m_magic != BASIS_MAGIC) return 0; - + if (format >= (int)transcoder_texture_format::cTFTotalTextureFormats) { assert(0); @@ -436,7 +504,7 @@ struct basis_file } const transcoder_texture_format tex_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) { @@ -454,7 +522,7 @@ struct basis_file if (m_magic != BASIS_MAGIC) return false; - return m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()) == basis_tex_format::cUASTC4x4; + return m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()) == basis_tex_format::cUASTC_LDR_4x4; } bool isETC1S() @@ -474,9 +542,9 @@ struct basis_file return false; basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); - return (fmt == basis_tex_format::cETC1S) || (fmt == basis_tex_format::cUASTC4x4); + return (fmt == basis_tex_format::cETC1S) || (fmt == basis_tex_format::cUASTC_LDR_4x4); } - + // True if the texture is UASTC HDR 4x4 or ASTC HDR 6x6. // In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions, currently 4x4 or 6x6), RGB9E5 or half-float RGB/RGBA images. bool isHDR() @@ -488,7 +556,7 @@ struct basis_file basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); return basis_tex_format_is_hdr(fmt); } - + bool isHDR4x4() { assert(m_magic == BASIS_MAGIC); @@ -506,9 +574,31 @@ struct basis_file return false; basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); - return (fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + return (fmt == basis_tex_format::cASTC_HDR_6x6) || (fmt == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE); } + // True for plain ASTC LDR 4x4-12x12 + bool isASTC_LDR() + { + assert(m_magic == BASIS_MAGIC); + if (m_magic != BASIS_MAGIC) + return false; + + basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); + return basis_tex_format_is_astc_ldr(fmt); + } + + // True for XUASTC LDR 4x4-12x12 + bool isXUASTC_LDR() + { + assert(m_magic == BASIS_MAGIC); + if (m_magic != BASIS_MAGIC) + return false; + + basis_tex_format fmt = m_transcoder.get_basis_tex_format(m_file.data(), m_file.size()); + return basis_tex_format_is_xuastc_ldr(fmt); + } + uint32_t startTranscoding() { assert(m_magic == BASIS_MAGIC); @@ -519,6 +609,7 @@ struct basis_file } // Here for backwards compat, prefer transcodeImageWithFlags(). + // format is transcoder_texture_format uint32_t transcodeImage(const emscripten::val& dst, uint32_t image_index, uint32_t level_index, uint32_t format, uint32_t unused, uint32_t get_alpha_for_opaque_formats) { (void)unused; @@ -531,7 +622,7 @@ struct basis_file return 0; const transcoder_texture_format transcoder_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) return 0; @@ -539,10 +630,11 @@ struct basis_file basisu::vector dst_data; uint32_t flags = get_alpha_for_opaque_formats ? cDecodeFlagsTranscodeAlphaDataToOpaqueFormats : 0; - + const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(image_index, level_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -568,15 +660,25 @@ struct basis_file flags); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + if (!dst_data.empty()) + { + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' is a JS Uint8Array + } +#endif + return status; } - - // Like transcodeImage(), but with fixed parameters. + + // Like transcodeImage(), but with updated parameters. // For flags, see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + // format is transcoder_texture_format uint32_t transcodeImageWithFlags(const emscripten::val& dst, uint32_t image_index, uint32_t level_index, uint32_t format, uint32_t flags) { assert(m_magic == BASIS_MAGIC); @@ -587,16 +689,17 @@ struct basis_file return 0; const transcoder_texture_format transcoder_format = static_cast(format); - + uint32_t orig_width, orig_height, total_src_blocks; if (!m_transcoder.get_image_level_desc(m_file.data(), m_file.size(), image_index, level_index, orig_width, orig_height, total_src_blocks)) return 0; basisu::vector dst_data; - + const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(image_index, level_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -622,10 +725,16 @@ struct basis_file flags); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' is a JS Uint8Array +#endif + return status; } }; @@ -658,10 +767,7 @@ struct ktx2_file bool m_is_valid = false; ktx2_file(const emscripten::val& jsBuffer) - : m_file([&]() { - size_t byteLength = jsBuffer["byteLength"].as(); - return basisu::vector(byteLength); - }()) + : m_file(jsBuffer["byteLength"].as()) { if (!g_basis_initialized_flag) { @@ -672,10 +778,16 @@ struct ktx2_file return; } +#if 0 unsigned int length = jsBuffer["length"].as(); emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = jsBuffer["constructor"].new_(memory, reinterpret_cast(m_file.data()), length); memoryView.call("set", jsBuffer); +#else + const size_t n = jsBuffer["byteLength"].as(); + emscripten::val dstView = emscripten::val(emscripten::typed_memory_view(n, m_file.data())); + dstView.call("set", jsBuffer); +#endif if (!m_transcoder.init(m_file.data(), m_file.size())) { @@ -764,8 +876,8 @@ struct ktx2_file hdr.m_kvd_byte_offset = h.m_kvd_byte_offset; hdr.m_kvd_byte_length = h.m_kvd_byte_length; - hdr.m_sgd_byte_offset = (uint32_t)h.m_sgd_byte_offset.get_uint64(); - hdr.m_sgd_byte_length = (uint32_t)h.m_sgd_byte_length.get_uint64(); + hdr.m_sgd_byte_offset = static_cast(h.m_sgd_byte_offset.get_uint64()); + hdr.m_sgd_byte_length = static_cast(h.m_sgd_byte_length.get_uint64()); return hdr; } @@ -823,6 +935,7 @@ struct ktx2_file return 1; } + // The image's original width, i.e. before being potentially expanded up to blocks. uint32_t getWidth() { assert(m_magic == KTX2_MAGIC); @@ -831,6 +944,7 @@ struct ktx2_file return m_transcoder.get_width(); } + // The image's original height, i.e. before being potentially expanded up to blocks. uint32_t getHeight() { assert(m_magic == KTX2_MAGIC); @@ -839,6 +953,7 @@ struct ktx2_file return m_transcoder.get_height(); } + // 4-12 uint32_t getBlockWidth() { assert(m_magic == KTX2_MAGIC); @@ -847,6 +962,7 @@ struct ktx2_file return m_transcoder.get_block_width(); } + // 4-12 uint32_t getBlockHeight() { assert(m_magic == KTX2_MAGIC); @@ -855,6 +971,7 @@ struct ktx2_file return m_transcoder.get_block_height(); } + // 2D or cubemaps uint32_t getFaces() { assert(m_magic == KTX2_MAGIC); @@ -863,6 +980,7 @@ struct ktx2_file return m_transcoder.get_faces(); } + // Layers for tex arrays uint32_t getLayers() { assert(m_magic == KTX2_MAGIC); @@ -871,6 +989,7 @@ struct ktx2_file return m_transcoder.get_layers(); } + // Mip-map levels uint32_t getLevels() { assert(m_magic == KTX2_MAGIC); @@ -879,7 +998,7 @@ struct ktx2_file return m_transcoder.get_levels(); } - // Returns a basis_tex_format: cETC1S, cUASTC4x4, or cUASTC_HDR_4x4, etc. + // Returns a basis_tex_format: cETC1S, cUASTC_LDR_4x4, or cUASTC_HDR_4x4, etc. - see basisu_file_headers.h uint32_t getBasisTexFormat() { assert(m_magic == KTX2_MAGIC); @@ -922,7 +1041,7 @@ struct ktx2_file return m_transcoder.is_etc1s(); } - // Returns true if the texture is UASTC HDR or ASTC HDR. In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions), RGB9E5 or half-float RGB/RGBA images. + // Returns true if the texture is UASTC HDR or ASTC HDR. In this case, it can only be transcoded to BC6H, ASTC HDR (of the same block dimensions), RGB9E5 or half-float RGB/RGBA images. bool isHDR() { assert(m_magic == KTX2_MAGIC); @@ -947,6 +1066,22 @@ struct ktx2_file return m_transcoder.is_hdr_6x6(); } + bool isASTC_LDR() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return false; + return m_transcoder.is_astc_ldr(); + } + + bool isXUASTC_LDR() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return false; + return m_transcoder.is_xuastc_ldr(); + } + bool getHasAlpha() { assert(m_magic == KTX2_MAGIC); @@ -979,6 +1114,14 @@ struct ktx2_file return m_transcoder.get_dfd_transfer_func(); } + bool isSRGB() + { + assert(m_magic == KTX2_MAGIC); + if (m_magic != KTX2_MAGIC) + return 0; + return m_transcoder.is_srgb(); + } + uint32_t getDFDFlags() { assert(m_magic == KTX2_MAGIC); @@ -1020,6 +1163,7 @@ struct ktx2_file return m_transcoder.is_video(); } + // The linear light LDR->HDR upconversion multiplier used (def=100.0 nits) float getLDRHDRUpconversionNitMultiplier() { assert(m_magic == KTX2_MAGIC); @@ -1056,6 +1200,7 @@ struct ktx2_file return info; } + // format is transcoder_texture_format uint32_t getImageTranscodedSizeInBytes(uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format) { assert(m_magic == KTX2_MAGIC); @@ -1069,7 +1214,7 @@ struct ktx2_file } const transcoder_texture_format tex_format = static_cast(format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) { @@ -1094,6 +1239,7 @@ struct ktx2_file // Here for backwards compat, prefer transcodeImageWithFlags(). // get_alpha_for_opaque_formats defaults to false // channel0/channel1 default to -1 + // format is transcoder_texture_format uint32_t transcodeImage(const emscripten::val& dst, uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format, uint32_t get_alpha_for_opaque_formats, int channel0, int channel1) { assert(m_magic == KTX2_MAGIC); @@ -1107,7 +1253,7 @@ struct ktx2_file const uint32_t dst_block_width = basis_get_block_width(transcoder_format); const uint32_t dst_block_height = basis_get_block_height(transcoder_format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) return 0; @@ -1120,7 +1266,8 @@ struct ktx2_file const uint32_t transcoded_size_in_bytes = getImageTranscodedSizeInBytes(level_index, layer_index, face_index, format); - dst_data.resize(transcoded_size_in_bytes); + if (!dst_data.try_resize(transcoded_size_in_bytes)) + return 0; uint32_t status; @@ -1151,15 +1298,22 @@ struct ktx2_file nullptr); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // 'dst' must be a Uint8Array (or compatible TypedArray) +#endif + return status; } - + // like transcodeImage(), but with fixed parameters (includes flags) // For flags, see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + // format is transcoder_texture_format uint32_t transcodeImageWithFlags(const emscripten::val& dst, uint32_t level_index, uint32_t layer_index, uint32_t face_index, uint32_t format, uint32_t flags, int channel0, int channel1) { assert(m_magic == KTX2_MAGIC); @@ -1168,12 +1322,12 @@ struct ktx2_file if (format >= (int)transcoder_texture_format::cTFTotalTextureFormats) return 0; - + const transcoder_texture_format transcoder_format = static_cast(format); const uint32_t dst_block_width = basis_get_block_width(transcoder_format); const uint32_t dst_block_height = basis_get_block_height(transcoder_format); - + ktx2_image_level_info info; if (!m_transcoder.get_image_level_info(info, level_index, layer_index, face_index)) return 0; @@ -1215,10 +1369,16 @@ struct ktx2_file nullptr); } +#if 0 emscripten::val memory = emscripten::val::module_property("HEAP8")["buffer"]; emscripten::val memoryView = emscripten::val::global("Uint8Array").new_(memory, reinterpret_cast(dst_data.data()), dst_data.size()); - dst.call("set", memoryView); +#else + const size_t n = dst_data.size(); + emscripten::val srcView = emscripten::val(emscripten::typed_memory_view(n, dst_data.data())); + dst.call("set", srcView); // dst = JS Uint8Array +#endif + return status; } @@ -1234,10 +1394,19 @@ enum class ldr_image_type cJPGImage = 2 }; +enum xuastc_ldr_syntax +{ + cFullArith = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullArith, + cHybridArithZStd = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cHybridArithZStd, + cFullZStd = (int)basist::astc_ldr_t::xuastc_ldr_syntax::cFullZStd, + cTotal = 3 +}; + class basis_encoder { bool m_threading_enabled = false; uint32_t m_num_extra_worker_threads = 0; + float m_last_encode_mip0_rgba_psnr = 0.0f; public: basis_compressor_params m_params; @@ -1252,7 +1421,7 @@ public: m_num_extra_worker_threads = num_extra_worker_threads; } - // Only works for LDR inputs. + // Only valid for LDR inputs. bool set_slice_source_image(uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t src_image_width, uint32_t src_image_height, ldr_image_type img_type) { // Resize the source_images array if necessary @@ -1261,7 +1430,8 @@ public: // First copy the src image buffer to the heap. basisu::vector src_image_buf; - copy_from_jsbuffer(src_image_js_val, src_image_buf); + if (!copy_from_jsbuffer(src_image_js_val, src_image_buf)) + return false; // Now load the source image. image& src_img = m_params.m_source_images[slice_index]; @@ -1328,14 +1498,15 @@ public: hdr_image_type img_type, bool ldr_srgb_to_linear_conversion, float ldr_to_hdr_nit_multiplier) { assert(ldr_to_hdr_nit_multiplier > 0.0f); - + // Resize the source_images_hdr array if necessary if (slice_index >= m_params.m_source_images_hdr.size()) m_params.m_source_images_hdr.resize(slice_index + 1); // First copy the src image buffer to the heap. basisu::vector src_image_buf; - copy_from_jsbuffer(src_image_js_val, src_image_buf); + if (!copy_from_jsbuffer(src_image_js_val, src_image_buf)) + return false; // Now load the source image. imagef& src_img = m_params.m_source_images_hdr[slice_index]; @@ -1345,14 +1516,14 @@ public: if ((img_type == hdr_image_type::cHITPNGImage) || (img_type == hdr_image_type::cHITJPGImage)) { - // Because we're loading the image ourselves we need to add these tags so the UI knows how to tone map LDR upconverted outputs. + // Because we're loading the image ourselves we need to add these tags so the UI knows how to tone map LDR upconverted outputs. // Normally basis_compressor adds them when it loads the images itself from source files. basist::ktx2_add_key_value(m_params.m_ktx2_key_values, "LDRUpconversionMultiplier", fmt_string("{}", ldr_to_hdr_nit_multiplier)); if (ldr_srgb_to_linear_conversion) basist::ktx2_add_key_value(m_params.m_ktx2_key_values, "LDRUpconversionSRGBToLinear", "1"); } - + return true; } @@ -1370,7 +1541,7 @@ public: // We don't use threading for now, but the compressor needs a job pool. uint32_t num_new_threads = 0; bool enable_threading = false; - + #if WASM_THREADS_ENABLED if ((emscripten_has_threading_support()) && (m_threading_enabled) && (m_num_extra_worker_threads)) { @@ -1384,20 +1555,29 @@ public: // Initialize the compression parameters structure. This is the same structure that the command line tool fills in. basis_compressor_params ¶ms = m_params; - + // Check to see if we would risk running out of memory in 32-bit WASM. There's not much we can do about this limit until memory64 is available. uint64_t total_src_texels = 0; - + for (uint32_t i = 0; i < m_params.m_source_images.size(); i++) total_src_texels += m_params.m_source_images[i].get_total_pixels(); for (uint32_t i = 0; i < m_params.m_source_images_hdr.size(); i++) total_src_texels += m_params.m_source_images_hdr[i].get_total_pixels(); + + // Try to prevent running out of memory inside WASM. + uint32_t max_pixels_thresh = BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS; - if (total_src_texels > BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS) + // The simpler compressors need less temporary memory, so their threshold can be higher. + if (m_params.is_etc1s() || m_params.is_uastc_ldr_4x4() || m_params.is_uastc_hdr_4x4()) { - printf("ERROR: basis_encoder::encode(): The total number of source texels to compress is too large for 32-bit WASM (above BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS in basis_wrappers.cpp)." - "This is not a fundamental limitation of the library, but of WASM. Processing images this large risks running out of memory until WASM memory64 is available.\n"); + max_pixels_thresh = BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS_HIGHER_LIMIT; + } + + if (total_src_texels > max_pixels_thresh) + { + printf("ERROR: basis_encoder::encode(): The total number of source texels to compress %llu is greater than %u, which is likely too large for WASM (above BASISU_ENCODER_MAX_SOURCE_IMAGE_PIXELS in basis_wrappers.cpp).", + total_src_texels, max_pixels_thresh); return 0; } @@ -1438,6 +1618,17 @@ public: #endif return 0; } + + m_last_encode_mip0_rgba_psnr = 0.0f; + if (comp.get_stats().size()) + { + float psnr = comp.get_stats()[0].m_basis_rgba_avg_psnr; + + if (psnr == 0.0f) + psnr = comp.get_stats()[0].m_basis_rgb_avg_psnr; // HDR, not RGBA though + + m_last_encode_mip0_rgba_psnr = psnr; + } if (params.m_create_ktx2_file) { @@ -1458,6 +1649,11 @@ public: return (uint32_t)comp.get_output_basis_file().size(); } } + + float get_last_encode_mip0_rgba_psnr() const + { + return m_last_encode_mip0_rgba_psnr; + } }; #endif @@ -1474,8 +1670,10 @@ public: bool decode_palettes(uint32_t num_endpoints, const emscripten::val& endpoint_data, uint32_t num_selectors, const emscripten::val& selector_data) { basisu::vector temp_endpoint_data, temp_selector_data; - copy_from_jsbuffer(endpoint_data, temp_endpoint_data); - copy_from_jsbuffer(selector_data, temp_selector_data); + if (!copy_from_jsbuffer(endpoint_data, temp_endpoint_data)) + return false; + if (!copy_from_jsbuffer(selector_data, temp_selector_data)) + return false; #if 0 printf("decode_palettes: %u %u %u %u, %u %u\n", @@ -1500,7 +1698,8 @@ public: bool decode_tables(const emscripten::val& table_data) { basisu::vector temp_table_data; - copy_from_jsbuffer(table_data, temp_table_data); + if (!copy_from_jsbuffer(table_data, temp_table_data)) + return false; if (!temp_table_data.size()) { @@ -1529,7 +1728,7 @@ public: if (!g_basis_initialized_flag) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: basis_init() must be called first\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: basis_init() must be called first\n"); #endif assert(0); return false; @@ -1537,12 +1736,13 @@ public: // FIXME: Access the JavaScript buffer directly vs. copying it. basisu::vector temp_comp_data; - copy_from_jsbuffer(compressed_data, temp_comp_data); + if (!copy_from_jsbuffer(compressed_data, temp_comp_data)) + return false; if (!temp_comp_data.size()) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: compressed_data is empty\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: compressed_data is empty\n"); #endif assert(0); return false; @@ -1552,7 +1752,7 @@ public: if (!output_blocks_len) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: output_blocks is empty\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: output_blocks is empty\n"); #endif assert(0); return false; @@ -1576,7 +1776,7 @@ public: if (!status) { #if BASISU_DEBUG_PRINTF - printf("transcode_etc1s_image: basisu_lowlevel_etc1s_transcoder::transcode_image failed\n"); + printf("lowlevel_etc1s_image_transcoder::transcode_image: basisu_lowlevel_etc1s_transcoder::transcode_image failed\n"); #endif assert(0); return false; @@ -1590,12 +1790,12 @@ public: }; // Supports UASTC LDR 4x4, UASTC HDR 4x4, and ASTC HDR 6x6/intermediate (but not ETC1S). -bool transcode_uastc_image( - uint32_t basis_tex_format_int, +bool transcode_uastc_image2( + uint32_t basis_tex_format_int, bool use_astc_srgb_decode_profile, uint32_t target_format_int, // see transcoder_texture_format const emscripten::val& output_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, const emscripten::val& compressed_data, - uint32_t num_blocks_x, uint32_t num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, uint32_t slice_offset, uint32_t slice_length, uint32_t decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. bool has_alpha, @@ -1607,13 +1807,13 @@ bool transcode_uastc_image( assert(basis_tex_format_int < (uint32_t)basis_tex_format::cTotalFormats); assert(target_format_int < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + basis_tex_format src_tex_format = static_cast(basis_tex_format_int); transcoder_texture_format target_format = static_cast(target_format_int); - basis_tex_format tex_format = static_cast(basis_tex_format_int); - + if (!g_basis_initialized_flag) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: basis_init() must be called first\n"); + printf("transcode_uastc_image2: basis_init() must be called first\n"); #endif assert(0); return false; @@ -1621,12 +1821,13 @@ bool transcode_uastc_image( // FIXME: Access the JavaScript buffer directly vs. copying it. basisu::vector temp_comp_data; - copy_from_jsbuffer(compressed_data, temp_comp_data); + if (!copy_from_jsbuffer(compressed_data, temp_comp_data)) + return false; if (!temp_comp_data.size()) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: compressed_data is empty\n"); + printf("transcode_uastc_image2: compressed_data is empty\n"); #endif assert(0); return false; @@ -1636,7 +1837,7 @@ bool transcode_uastc_image( if (!output_blocks_len) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: output_blocks is empty\n"); + printf("transcode_uastc_image2: output_blocks is empty\n"); #endif assert(0); return false; @@ -1646,7 +1847,7 @@ bool transcode_uastc_image( printf("format: %u\n", (uint32_t)target_format); printf("output_blocks size: %u buf size: %u\n", output_blocks_len, output_blocks_buf_size_in_blocks_or_pixels); printf("compressed_data size: %u\n", compressed_data["byteLength"].as()); - printf("%u %u %u %u %u\n", num_blocks_x, num_blocks_y, orig_width, orig_height, level_index); + printf("%u %u %u %u %u\n", src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index); printf("%u %u\n", slice_offset, slice_length); printf("%u\n", decode_flags); printf("has_alpha: %u is_video: %u\n", has_alpha, is_video); @@ -1654,8 +1855,27 @@ bool transcode_uastc_image( basisu::vector temp_output_blocks(output_blocks_len); - bool status = false; - if (tex_format == basis_tex_format::cUASTC_HDR_4x4) + bool status = false; + if (basis_tex_format_is_astc_ldr(src_tex_format) || basis_tex_format_is_xuastc_ldr(src_tex_format)) + { + basisu_lowlevel_xuastc_ldr_transcoder transcoder; + + status = transcoder.transcode_image( + src_tex_format, use_astc_srgb_decode_profile, + (transcoder_texture_format)target_format, + &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, + &temp_comp_data[0], temp_comp_data.size(), + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, + slice_offset, slice_length, + decode_flags, + has_alpha, + is_video, + output_row_pitch_in_blocks_or_pixels, + nullptr, + output_rows_in_pixels, + channel0, channel1); + } + else if (src_tex_format == basis_tex_format::cUASTC_HDR_4x4) { basisu_lowlevel_uastc_hdr_4x4_transcoder transcoder; @@ -1663,7 +1883,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1673,7 +1893,7 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cASTC_HDR_6x6) + else if (src_tex_format == basis_tex_format::cASTC_HDR_6x6) { basisu_lowlevel_astc_hdr_6x6_transcoder transcoder; @@ -1681,7 +1901,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1691,15 +1911,15 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + else if (src_tex_format == basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) { - basisu_lowlevel_astc_hdr_6x6_intermediate_transcoder transcoder; + basisu_lowlevel_uastc_hdr_6x6_intermediate_transcoder transcoder; status = transcoder.transcode_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1709,7 +1929,7 @@ bool transcode_uastc_image( output_rows_in_pixels, channel0, channel1); } - else if (tex_format == basis_tex_format::cUASTC4x4) + else if (src_tex_format == basis_tex_format::cUASTC_LDR_4x4) { basisu_lowlevel_uastc_ldr_4x4_transcoder transcoder; @@ -1717,7 +1937,7 @@ bool transcode_uastc_image( (transcoder_texture_format)target_format, &temp_output_blocks[0], output_blocks_buf_size_in_blocks_or_pixels, &temp_comp_data[0], temp_comp_data.size(), - num_blocks_x, num_blocks_y, orig_width, orig_height, level_index, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, slice_offset, slice_length, decode_flags, has_alpha, @@ -1735,7 +1955,7 @@ bool transcode_uastc_image( if (!status) { #if BASISU_DEBUG_PRINTF - printf("transcode_uastc_image: basisu_lowlevel_uastc_transcoder::transcode_image failed\n"); + printf("transcode_uastc_image2: basisu_lowlevel_uastc_transcoder::transcode_image failed\n"); #endif assert(0); return false; @@ -1747,82 +1967,154 @@ bool transcode_uastc_image( return true; } +// Previous API - prefer transcode_uastc_image2(), which allows the caller to control the ASTC decode profile (srgb/linear) for XUASTC/ASTC. +bool transcode_uastc_image( + uint32_t basis_tex_format_int, + uint32_t target_format_int, // see transcoder_texture_format + const emscripten::val& output_blocks, uint32_t output_blocks_buf_size_in_blocks_or_pixels, + const emscripten::val& compressed_data, + uint32_t src_num_blocks_x, uint32_t src_num_blocks_y, uint32_t orig_width, uint32_t orig_height, uint32_t level_index, + uint32_t slice_offset, uint32_t slice_length, + uint32_t decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + bool has_alpha, + bool is_video, + uint32_t output_row_pitch_in_blocks_or_pixels, + uint32_t output_rows_in_pixels, + int channel0, int channel1) +{ + // Just assume sRGB decode profile - which is the compressor's default. + const bool use_astc_srgb_decode_profile = true; + + return transcode_uastc_image2( + basis_tex_format_int, use_astc_srgb_decode_profile, + target_format_int, // see transcoder_texture_format + output_blocks, output_blocks_buf_size_in_blocks_or_pixels, + compressed_data, + src_num_blocks_x, src_num_blocks_y, orig_width, orig_height, level_index, + slice_offset, slice_length, + decode_flags, // see cDecodeFlagsPVRTCDecodeToNextPow2 etc. + has_alpha, + is_video, + output_row_pitch_in_blocks_or_pixels, + output_rows_in_pixels, + channel0, channel1); +} + +// transcoder_tex_fmt is transcoder_texture_format uint32_t get_bytes_per_block_or_pixel(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_bytes_per_block_or_pixel(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_has_alpha(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_has_alpha(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_hdr(uint32_t transcode_tex_fmt) { assert(transcode_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_is_hdr(static_cast(transcode_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_ldr(uint32_t transcode_tex_fmt) { assert(transcode_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return !basis_transcoder_format_is_hdr(static_cast(transcode_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format bool format_is_uncompressed(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_transcoder_format_is_uncompressed(static_cast(transcoder_tex_fmt)); } +// transcoder_tex_fmt is transcoder_texture_format, file_fmt is basis_tex_fmt bool is_format_supported(uint32_t transcoder_tex_fmt, uint32_t file_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); return basis_is_format_supported(static_cast(transcoder_tex_fmt), static_cast(file_fmt)); } -// transcoder_texture_format +// transcoder_tex_fmt is transcoder_texture_format uint32_t get_format_block_width(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_block_width(static_cast(transcoder_tex_fmt)); } -// transcoder_texture_format +// fmt is transcoder_texture_format uint32_t get_format_block_height(uint32_t transcoder_tex_fmt) { assert(transcoder_tex_fmt < (uint32_t)transcoder_texture_format::cTFTotalTextureFormats); return basis_get_block_height(static_cast(transcoder_tex_fmt)); } -// basis_tex_format -uint32_t get_basis_tex_format_block_width(uint32_t fmt) +// file_fmt is basis_tex_format +uint32_t get_basis_tex_format_block_width(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return basis_tex_format_get_block_width(static_cast(fmt)); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_get_block_width(static_cast(file_fmt)); } -// basis_tex_format -uint32_t get_basis_tex_format_block_height(uint32_t fmt) +// file_fmt is basis_tex_format +uint32_t get_basis_tex_format_block_height(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return basis_tex_format_get_block_height(static_cast(fmt)); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_get_block_height(static_cast(file_fmt)); } -// basis_tex_format -bool is_basis_tex_format_hdr(uint32_t fmt) +// file_fmt is basis_tex_format +bool is_basis_tex_format_hdr(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return ((basis_tex_format)fmt == basis_tex_format::cUASTC_HDR_4x4) || ((basis_tex_format)fmt == basis_tex_format::cASTC_HDR_6x6) || ((basis_tex_format)fmt == basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_hdr((basis_tex_format)file_fmt); } -// basis_tex_format -bool is_basis_tex_format_ldr(uint32_t fmt) +// file_fmt is basis_tex_format +bool is_basis_tex_format_ldr(uint32_t file_fmt) { - assert(fmt < (uint32_t)basis_tex_format::cTotalFormats); - return ((basis_tex_format)fmt == basis_tex_format::cETC1S) || ((basis_tex_format)fmt == basis_tex_format::cUASTC4x4); + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_ldr((basis_tex_format)file_fmt); +} + +// file_fmt is basis_tex_format +bool is_basis_tex_format_xuastc_ldr(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_xuastc_ldr((basis_tex_format)file_fmt); +} + +// file_fmt is basis_tex_format +bool is_basis_tex_format_astc_ldr(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return basis_tex_format_is_astc_ldr((basis_tex_format)file_fmt); +} + +// Returns transcoder_texture_format, file_fmt is basis_tex_format. +// // Returns the best ASTC texture format to use given any basis_tex_format (the one with the proper block size). +// Use get_transcoder_texture_format_from_basis_tex_format() instead (same thing). Here for backwards compat. +uint32_t get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return (uint32_t)basis_get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(static_cast(file_fmt)); +} + +// Same as get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format(), just a smaller name, works with any basis_tex_format. +// Returns the best ASTC texture format to use given any basis_tex_format (the one with the proper block size). +// Returns transcoder_texture_format, file_fmt is basis_tex_format. +uint32_t get_transcoder_texture_format_from_basis_tex_format(uint32_t file_fmt) +{ + assert(file_fmt < (uint32_t)basis_tex_format::cTotalFormats); + return (uint32_t)basis_get_transcoder_texture_format_from_basis_tex_format(static_cast(file_fmt)); } uint32_t convert_float_to_half(float f) @@ -1848,10 +2140,10 @@ uint32_t get_debug_flags_wrapper() EMSCRIPTEN_BINDINGS(basis_codec) { function("initializeBasis", &basis_init); - + function("setDebugFlags", &set_debug_flags_wrapper); function("getDebugFlags", &get_debug_flags_wrapper); - + // Expose BasisFileDesc structure value_object("BasisFileDesc") .field("version", &basis_file_desc::m_version) @@ -1926,6 +2218,19 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cTFRGBA_HALF", transcoder_texture_format::cTFRGBA_HALF) .value("cTFRGB_9E5", transcoder_texture_format::cTFRGB_9E5) .value("cTFASTC_HDR_6x6_RGBA", transcoder_texture_format::cTFASTC_HDR_6x6_RGBA) + .value("cTFASTC_LDR_5x4_RGBA", transcoder_texture_format::cTFASTC_LDR_5x4_RGBA) + .value("cTFASTC_LDR_5x5_RGBA", transcoder_texture_format::cTFASTC_LDR_5x5_RGBA) + .value("cTFASTC_LDR_6x5_RGBA", transcoder_texture_format::cTFASTC_LDR_6x5_RGBA) + .value("cTFASTC_LDR_6x6_RGBA", transcoder_texture_format::cTFASTC_LDR_6x6_RGBA) + .value("cTFASTC_LDR_8x5_RGBA", transcoder_texture_format::cTFASTC_LDR_8x5_RGBA) + .value("cTFASTC_LDR_8x6_RGBA", transcoder_texture_format::cTFASTC_LDR_8x6_RGBA) + .value("cTFASTC_LDR_10x5_RGBA", transcoder_texture_format::cTFASTC_LDR_10x5_RGBA) + .value("cTFASTC_LDR_10x6_RGBA", transcoder_texture_format::cTFASTC_LDR_10x6_RGBA) + .value("cTFASTC_LDR_8x8_RGBA", transcoder_texture_format::cTFASTC_LDR_8x8_RGBA) + .value("cTFASTC_LDR_10x8_RGBA", transcoder_texture_format::cTFASTC_LDR_10x8_RGBA) + .value("cTFASTC_LDR_10x10_RGBA", transcoder_texture_format::cTFASTC_LDR_10x10_RGBA) + .value("cTFASTC_LDR_12x10_RGBA", transcoder_texture_format::cTFASTC_LDR_12x10_RGBA) + .value("cTFASTC_LDR_12x12_RGBA", transcoder_texture_format::cTFASTC_LDR_12x12_RGBA) .value("cTFTotalTextureFormats", transcoder_texture_format::cTFTotalTextureFormats) ; @@ -1944,7 +2249,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { function("isBasisTexFormatHDR", &is_basis_tex_format_hdr); function("isBasisTexFormatLDR", &is_basis_tex_format_ldr); - + function("isBasisTexFormatXUASTCLDR", &is_basis_tex_format_xuastc_ldr); + function("isBasisTexFormatASTCLDR", &is_basis_tex_format_astc_ldr); + function("getTranscoderTextureFormatFromXUASTCOrASTCLDRBasisTexFormat", &get_transcoder_texture_format_from_xuastc_or_astc_ldr_basis_tex_format); + function("getTranscoderTextureFormatFromBasisTexFormat", &get_transcoder_texture_format_from_basis_tex_format); + function("convertFloatToHalf", &convert_float_to_half); function("convertHalfToFloat", &convert_half_to_float); @@ -1957,13 +2266,46 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cBASISTexTypeVolume", cBASISTexTypeVolume) ; - // Expose enum basis_tex_format + // Expose enum basis_tex_format - supported KTX2/.basis texture types. enum_("basis_tex_format") .value("cETC1S", basis_tex_format::cETC1S) - .value("cUASTC4x4", basis_tex_format::cUASTC4x4) + .value("cUASTC4x4", basis_tex_format::cUASTC_LDR_4x4) // name has changed, keeping for backwards compat + .value("cUASTC_LDR_4x4", basis_tex_format::cUASTC_LDR_4x4) .value("cUASTC_HDR_4x4", basis_tex_format::cUASTC_HDR_4x4) .value("cASTC_HDR_6x6", basis_tex_format::cASTC_HDR_6x6) - .value("cASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cASTC_HDR_6x6_INTERMEDIATE) + .value("cUASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) + .value("cUASTC_HDR_6x6", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) // the correct name + .value("cASTC_HDR_6x6_INTERMEDIATE", basis_tex_format::cUASTC_HDR_6x6_INTERMEDIATE) // was misnamed in previous release, keeping for backwards compat + // XUASTC LDR 4x4-12x12 + .value("cXUASTC_LDR_4x4", basis_tex_format::cXUASTC_LDR_4x4) + .value("cXUASTC_LDR_5x4", basis_tex_format::cXUASTC_LDR_5x4) + .value("cXUASTC_LDR_5x5", basis_tex_format::cXUASTC_LDR_5x5) + .value("cXUASTC_LDR_6x5", basis_tex_format::cXUASTC_LDR_6x5) + .value("cXUASTC_LDR_6x6", basis_tex_format::cXUASTC_LDR_6x6) + .value("cXUASTC_LDR_8x5", basis_tex_format::cXUASTC_LDR_8x5) + .value("cXUASTC_LDR_8x6", basis_tex_format::cXUASTC_LDR_8x6) + .value("cXUASTC_LDR_10x5", basis_tex_format::cXUASTC_LDR_10x5) + .value("cXUASTC_LDR_10x6", basis_tex_format::cXUASTC_LDR_10x6) + .value("cXUASTC_LDR_8x8", basis_tex_format::cXUASTC_LDR_8x8) + .value("cXUASTC_LDR_10x8", basis_tex_format::cXUASTC_LDR_10x8) + .value("cXUASTC_LDR_10x10", basis_tex_format::cXUASTC_LDR_10x10) + .value("cXUASTC_LDR_12x10", basis_tex_format::cXUASTC_LDR_12x10) + .value("cXUASTC_LDR_12x12", basis_tex_format::cXUASTC_LDR_12x12) + // ASTC LDR 4x4-12x12 + .value("cASTC_LDR_4x4", basis_tex_format::cASTC_LDR_4x4) + .value("cASTC_LDR_5x4", basis_tex_format::cASTC_LDR_5x4) + .value("cASTC_LDR_5x5", basis_tex_format::cASTC_LDR_5x5) + .value("cASTC_LDR_6x5", basis_tex_format::cASTC_LDR_6x5) + .value("cASTC_LDR_6x6", basis_tex_format::cASTC_LDR_6x6) + .value("cASTC_LDR_8x5", basis_tex_format::cASTC_LDR_8x5) + .value("cASTC_LDR_8x6", basis_tex_format::cASTC_LDR_8x6) + .value("cASTC_LDR_10x5", basis_tex_format::cASTC_LDR_10x5) + .value("cASTC_LDR_10x6", basis_tex_format::cASTC_LDR_10x6) + .value("cASTC_LDR_8x8", basis_tex_format::cASTC_LDR_8x8) + .value("cASTC_LDR_10x8", basis_tex_format::cASTC_LDR_10x8) + .value("cASTC_LDR_10x10", basis_tex_format::cASTC_LDR_10x10) + .value("cASTC_LDR_12x10", basis_tex_format::cASTC_LDR_12x10) + .value("cASTC_LDR_12x12", basis_tex_format::cASTC_LDR_12x12) ; // .basis file transcoder object. If all you want to do is transcode already encoded .basis files, this is all you really need. @@ -1999,6 +2341,12 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("isLDR", optional_override([](basis_file& self) { return self.isLDR(); })) + .function("isASTC_LDR", optional_override([](basis_file& self) { + return self.isASTC_LDR(); + })) + .function("isXUASTC_LDR", optional_override([](basis_file& self) { + return self.isXUASTC_LDR(); + })) .function("getNumImages", optional_override([](basis_file& self) { return self.getNumImages(); })) @@ -2055,6 +2403,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cDecodeFlagsOutputHasAlphaIndices", cDecodeFlagsOutputHasAlphaIndices) .value("cDecodeFlagsHighQuality", cDecodeFlagsHighQuality) .value("cDecodeFlagsNoETC1SChromaFiltering", cDecodeFlagsNoETC1SChromaFiltering) + .value("cDecodeFlagsNoDeblockFiltering", cDecodeFlagsNoDeblockFiltering) + .value("cDecodeFlagsStrongerDeblockFiltering", cDecodeFlagsStrongerDeblockFiltering) + .value("cDecodeFlagsForceDeblockFiltering", cDecodeFlagsForceDeblockFiltering) + .value("cDecodeFlagXUASTCLDRDisableFastBC7Transcoding", cDecodeFlagXUASTCLDRDisableFastBC7Transcoding) ; // The low-level ETC1S transcoder is a class because it has persistent state (such as the endpoint/selector codebooks and Huffman tables, and transcoder state for video) @@ -2067,10 +2419,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { // The low-level UASTC transcoder (for UASTC LDR 4x4, HDR 4x4, or ASTC HDR 6x6) is a single function. function("transcodeUASTCImage", &transcode_uastc_image); + function("transcodeUASTCImage2", &transcode_uastc_image2); function("transcoderSupportsKTX2", &basisu_transcoder_supports_ktx2); function("transcoderSupportsKTX2Zstd", &basisu_transcoder_supports_ktx2_zstd); - + #if BASISD_SUPPORT_KTX2 // KTX2 enums/constants enum_("ktx2_supercompression") @@ -2080,10 +2433,11 @@ EMSCRIPTEN_BINDINGS(basis_codec) { ; constant("KTX2_VK_FORMAT_UNDEFINED", KTX2_VK_FORMAT_UNDEFINED); - constant("KTX2_KDF_DF_MODEL_UASTC", KTX2_KDF_DF_MODEL_UASTC_LDR_4X4); constant("KTX2_KDF_DF_MODEL_ETC1S", KTX2_KDF_DF_MODEL_ETC1S); - constant("KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE", KTX2_KDF_DF_MODEL_ASTC_HDR_6X6_INTERMEDIATE); - + constant("KTX2_KDF_DF_MODEL_UASTC", KTX2_KDF_DF_MODEL_UASTC_LDR_4X4); + constant("KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE", KTX2_KDF_DF_MODEL_UASTC_HDR_6X6_INTERMEDIATE); + constant("KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE", KTX2_KDF_DF_MODEL_XUASTC_LDR_INTERMEDIATE); + constant("KTX2_IMAGE_IS_P_FRAME", KTX2_IMAGE_IS_P_FRAME); constant("KTX2_UASTC_BLOCK_SIZE", KTX2_UASTC_BLOCK_SIZE); constant("KTX2_MAX_SUPPORTED_LEVEL_COUNT", KTX2_MAX_SUPPORTED_LEVEL_COUNT); @@ -2185,10 +2539,13 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("isHDR6x6", &ktx2_file::isHDR6x6) .function("isLDR", &ktx2_file::isLDR) .function("isETC1S", &ktx2_file::isETC1S) + .function("isASTC_LDR", &ktx2_file::isASTC_LDR) + .function("isXUASTC_LDR", &ktx2_file::isXUASTC_LDR) .function("getHasAlpha", &ktx2_file::getHasAlpha) .function("getDFDColorModel", &ktx2_file::getDFDColorModel) .function("getDFDColorPrimaries", &ktx2_file::getDFDColorPrimaries) .function("getDFDTransferFunc", &ktx2_file::getDFDTransferFunc) + .function("isSRGB", &ktx2_file::isSRGB) .function("getDFDFlags", &ktx2_file::getDFDFlags) .function("getDFDTotalSamples", &ktx2_file::getDFDTotalSamples) .function("getDFDChannelID0", &ktx2_file::getDFDChannelID0) @@ -2219,6 +2576,8 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("BASISU_MAX_IMAGE_DIMENSION", BASISU_MAX_IMAGE_DIMENSION); constant("BASISU_QUALITY_MIN", BASISU_QUALITY_MIN); constant("BASISU_QUALITY_MAX", BASISU_QUALITY_MAX); + constant("BASISU_XUASTC_QUALITY_MIN", BASISU_XUASTC_QUALITY_MIN); + constant("BASISU_XUASTC_QUALITY_MAX", BASISU_XUASTC_QUALITY_MAX); constant("BASISU_MAX_ENDPOINT_CLUSTERS", BASISU_MAX_ENDPOINT_CLUSTERS); constant("BASISU_MAX_SELECTOR_CLUSTERS", BASISU_MAX_SELECTOR_CLUSTERS); constant("BASISU_MAX_SLICES", BASISU_MAX_SLICES); @@ -2226,10 +2585,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("BASISU_RDO_UASTC_DICT_SIZE_MIN", BASISU_RDO_UASTC_DICT_SIZE_MIN); constant("BASISU_RDO_UASTC_DICT_SIZE_MAX", BASISU_RDO_UASTC_DICT_SIZE_MAX); constant("BASISU_MAX_RESAMPLER_FILTERS", g_num_resample_filters); - constant("BASISU_DEFAULT_COMPRESSION_LEVEL", BASISU_DEFAULT_COMPRESSION_LEVEL); - constant("BASISU_MAX_COMPRESSION_LEVEL", BASISU_MAX_COMPRESSION_LEVEL); + constant("BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL", BASISU_DEFAULT_ETC1S_COMPRESSION_LEVEL); + constant("BASISU_MAX_ETC1S_COMPRESSION_LEVEL", BASISU_MAX_ETC1S_COMPRESSION_LEVEL); - // The maximum representable floating point value in a UASTC HDR or ASTC HDR texture (any larger values will get clamped and a warning issued). + // The maximum representable floating point value in a UASTC HDR or ASTC HDR texture (any larger values will get clamped and a warning issued). constant("ASTC_HDR_MAX_VAL", basist::ASTC_HDR_MAX_VAL); // UASTC LDR/HDR flags/options @@ -2244,12 +2603,12 @@ EMSCRIPTEN_BINDINGS(basis_codec) { constant("cPackUASTCETC1FasterHints", cPackUASTCETC1FasterHints); constant("cPackUASTCETC1FastestHints", cPackUASTCETC1FastestHints); constant("cPackUASTCETC1DisableFlipAndIndividual", cPackUASTCETC1DisableFlipAndIndividual); - + constant("UASTC_RDO_DEFAULT_MAX_ALLOWED_RMS_INCREASE_RATIO", UASTC_RDO_DEFAULT_MAX_ALLOWED_RMS_INCREASE_RATIO); constant("UASTC_RDO_DEFAULT_SKIP_BLOCK_RMS_THRESH", UASTC_RDO_DEFAULT_SKIP_BLOCK_RMS_THRESH); constant("cPackASTC6x6MaxUserCompLevel", ::astc_6x6_hdr::ASTC_HDR_6X6_MAX_USER_COMP_LEVEL); - + enum_("hdr_image_type") .value("cHITRGBAHalfFloat", hdr_image_type::cHITRGBAHalfFloat) .value("cHITRGBAFloat", hdr_image_type::cHITRGBAFloat) @@ -2265,6 +2624,13 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .value("cJPGImage", ldr_image_type::cJPGImage) ; + enum_("xuastc_ldr_syntax") + .value("cFullArith", xuastc_ldr_syntax::cFullArith) + .value("cHybridArithZStd", xuastc_ldr_syntax::cHybridArithZStd) + .value("cFullZStd", xuastc_ldr_syntax::cFullZStd) + .value("cTotal", xuastc_ldr_syntax::cTotal) + ; + // Compression/encoding object. // You create this object, call the set() methods to fill in the parameters/source images/options, call encode(), and you get back a .basis or .KTX2 file. // You can call .encode() multiple times, changing the parameters/options in between calls. @@ -2278,6 +2644,10 @@ EMSCRIPTEN_BINDINGS(basis_codec) { return self.encode(dst_basis_file_js_val); })) + .function("getLastEncodeMip0RGBAPSNR", optional_override([](basis_encoder& self) { + return self.get_last_encode_mip0_rgba_psnr(); + })) + // Sets the slice's source image, either from a PNG/JPG file or from a raw 32-bit RGBA raster image. // If the input is a raster image, the buffer must be width*height*4 bytes in size. The raster image is stored in top down scanline order. // The first texel is the top-left texel. The texel byte order in memory is R,G,B,A (R first at offset 0, A last at offset 3). @@ -2287,18 +2657,19 @@ EMSCRIPTEN_BINDINGS(basis_codec) { return self.set_slice_source_image(slice_index, src_image_js_val, width, height, (ldr_image_type)img_type); })) + // If true threaded compression will be used with X *extra* helper threads. .function("controlThreading", optional_override([](basis_encoder& self, bool enable_threading, uint32_t num_extra_worker_threads) { return self.control_threading(enable_threading, num_extra_worker_threads); })) // HDR targets only - .function("setSliceSourceImageHDR", optional_override([](basis_encoder& self, uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t width, uint32_t height, uint32_t img_type, + .function("setSliceSourceImageHDR", optional_override([](basis_encoder& self, uint32_t slice_index, const emscripten::val& src_image_js_val, uint32_t width, uint32_t height, uint32_t img_type, bool ldr_srgb_to_linear_conversion, float ldr_to_hdr_nit_multiplier) { return self.set_slice_source_image_hdr(slice_index, src_image_js_val, width, height, (hdr_image_type)img_type, ldr_srgb_to_linear_conversion, ldr_to_hdr_nit_multiplier); })) - // Sets the desired encoding format. This is the preferred way to control which format the encoder creates. - // tex_format is a basis_tex_format (cETC1s, cUASTC4x4, cUASTC_HDR_4x4 etc.) + // Sets the desired encoding format. This is the preferred way to control which format/ASTC block size the encoder creates. + // tex_format is a basis_tex_format (cETC1s, cUASTC_LDR_4x4, cUASTC_HDR_4x4 etc.) - see basisu_file_headers.h. // This can be used instead of the older setUASTC(), setHDR() etc. methods. // All formats .function("setFormatMode", optional_override([](basis_encoder& self, int tex_format) { @@ -2306,12 +2677,23 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.set_format_mode((basis_tex_format)tex_format); })) + // setFormatModeAndEffortQuality() is like setFormatMode(), except it also sets the effort [0,10] and quality [0,100] parameters to (hopefully) reasonable values for the selected format. + // If effort==-1, no effort related parameters will be modified. + // If quality==-1, no quality related parameters will be modified. + // These values directly correspond to the command line tool's "-effort X" and "-quality X" unified codec compression options. + .function("setFormatModeAndQualityEffort", optional_override([](basis_encoder& self, int tex_format, int quality, int effort, bool set_defaults) { + assert((tex_format >= 0) && (tex_format < (uint32_t)basis_tex_format::cTotalFormats)); + assert((effort >= -1) && (effort <= 10)); + assert((quality >= -1) && (quality <= 100)); + self.m_params.set_format_mode_and_quality_effort((basis_tex_format)tex_format, quality, effort, set_defaults); + })) + // If true, the encoder will output a UASTC LDR 4x4 texture, otherwise a ETC1S texture. // (This is for backwards compatibility, prefer setFormatMode() instead.) // All formats .function("setUASTC", optional_override([](basis_encoder& self, bool uastc_flag) { if (uastc_flag) - self.m_params.set_format_mode(basis_tex_format::cUASTC4x4); + self.m_params.set_format_mode(basis_tex_format::cUASTC_LDR_4x4); else self.m_params.set_format_mode(basis_tex_format::cETC1S); })) @@ -2325,8 +2707,9 @@ EMSCRIPTEN_BINDINGS(basis_codec) { else self.m_params.set_format_mode(basis_tex_format::cETC1S); // don't really know what to set })) - - // Sets the UASTC HDR 4x4 quality vs. encoder performance tradeoff (0-4, default is 1). Higher=slower but better quality. + + // Sets the UASTC HDR 4x4 quality/effort vs. encoder performance tradeoff (0-4, default is 1). Higher=slower but better quality. + // TODO: Rename, this is really a compressor "effort" level. // UASTC HDR 4x4 .function("setUASTCHDRQualityLevel", optional_override([](basis_encoder& self, int level) { assert((level >= uastc_hdr_4x4_codec_options::cMinLevel) && (level <= uastc_hdr_4x4_codec_options::cMaxLevel)); @@ -2348,6 +2731,7 @@ EMSCRIPTEN_BINDINGS(basis_codec) { // If true, the input is assumed to be in sRGB space. Be sure to set this correctly! (Examples: True on photos, albedo/spec maps, and false on normal maps.) // In HDR mode, if perceptual is true R and G are weighted higher (2.0, 3.0) than B (1.0). Otherwise the encoder uses equal weightings for each channel. + // Importantly, also see setKTX2SRGBTransferFunc() and setMipSRGB(). // ETC1S, UASTC LDR 4x4, UASTC HDR 4x4 .function("setPerceptual", optional_override([](basis_encoder& self, bool perceptual_flag) { self.m_params.m_perceptual = perceptual_flag; @@ -2382,8 +2766,9 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setLambda", optional_override([](basis_encoder& self, float rdo_quality) { self.m_params.m_astc_hdr_6x6_options.m_lambda = rdo_quality; })) - - // ASTC HDR 6x6: Enables REC 2020 delta E ITP vs. REC 709 in the encoder. + + // ASTC HDR 6x6: Enables REC 2020 delta E ITP vs. REC 709 in the encoder (and sets the colorspace in the KTX2 header). + // Note this colorspace always goes into the KTX2 header (DFD), for all modes (ETC1S, UASTC LDR 4x4, etc.) .function("setRec2020", optional_override([](basis_encoder& self, bool rec2020) { self.m_params.m_astc_hdr_6x6_options.m_rec2020_bt2100_color_gamut = rec2020; })) @@ -2471,22 +2856,23 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.m_etc1s_max_selector_clusters = max_selector_clusters; })) - // Sets the ETC1S encoder's quality level, which controls the file size vs. quality tradeoff. - // Default is -1 (meaning unused - the compressor will use m_max_endpoint_clusters/m_max_selector_clusters instead to control the codebook sizes). - // Range is [1,BASISU_QUALITY_MAX] - // ETC1S mode + // Sets the ETC1S or XUASTC LDR 4x4-12x12 encoder's quality level, which controls the file size vs. quality tradeoff. + // Default is -1 (meaning unused - the compressor will use m_max_endpoint_clusters/m_max_selector_clusters instead to control the codebook sizes in ETC1S mode, or no DCT in XUASTC LDR 4x4-12x12 mode). + // Range is [1,BASISU_QUALITY_MAX] (ETC1S) or [1,100] (XUASTC LDR 4x4-12x12) + // For XUASTC LDR, you also need to enable DCT usage, below. + // ETC1S mode or XUASTC LDR 4x4-12x12 .function("setQualityLevel", optional_override([](basis_encoder& self, int quality_level) { assert(quality_level >= -1 && quality_level <= BASISU_QUALITY_MAX); - self.m_params.m_etc1s_quality_level = quality_level; + self.m_params.m_quality_level = quality_level; })) // The compression_level parameter controls the encoder perf vs. file size tradeoff for ETC1S files. // It does not directly control file size vs. quality - see quality_level(). // Default is BASISU_DEFAULT_COMPRESSION_LEVEL, range is [0,BASISU_MAX_COMPRESSION_LEVEL] // ETC1S mode - .function("setCompressionLevel", optional_override([](basis_encoder& self, int comp_level) { - assert(comp_level >= 0 && comp_level <= BASISU_MAX_COMPRESSION_LEVEL); - self.m_params.m_compression_level = comp_level; + .function("setETC1SCompressionLevel", optional_override([](basis_encoder& self, int comp_level) { + assert(comp_level >= 0 && comp_level <= BASISU_MAX_ETC1S_COMPRESSION_LEVEL); + self.m_params.m_etc1s_compression_level = comp_level; })) // setNormalMapMode is the same as the basisu.exe "-normal_map" option. It tunes several codec parameters so compression works better on normal maps. @@ -2526,16 +2912,16 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setKTX2UASTCSupercompression", optional_override([](basis_encoder& self, bool use_zstandard) { self.m_params.m_ktx2_uastc_supercompression = use_zstandard ? basist::KTX2_SS_ZSTANDARD : basist::KTX2_SS_NONE; })) - - // KTX2: Use sRGB transfer func in the file's DFD. Default is FALSE. This should very probably match the "perceptual" setting. - // All formats - .function("setKTX2SRGBTransferFunc", optional_override([](basis_encoder& self, bool srgb_transfer_func) { - self.m_params.m_ktx2_srgb_transfer_func = srgb_transfer_func; - })) - + // TODO: Expose KTX2 key value array, other options to JavaScript. See encoder/basisu_comp.h. #endif + // KTX2/.basis: Use sRGB transfer func in the file's header/DFD. Default is FALSE. This should very probably match the "perceptual" and mipRGB settings. + // All formats + .function("setKTX2AndBasisSRGBTransferFunc", optional_override([](basis_encoder& self, bool srgb_transfer_func) { + self.m_params.m_ktx2_and_basis_srgb_transfer_function = srgb_transfer_func; + })) + // --- Mip-map options (format independent) // If true mipmaps will be generated from the source images @@ -2621,7 +3007,7 @@ EMSCRIPTEN_BINDINGS(basis_codec) { .function("setRDOUASTCQualityScalar", optional_override([](basis_encoder& self, float rdo_quality) { self.m_params.m_rdo_uastc_ldr_4x4_quality_scalar = rdo_quality; })) - + // Default is BASISU_RDO_UASTC_DICT_SIZE_DEFAULT, range is [BASISU_RDO_UASTC_DICT_SIZE_MIN, BASISU_RDO_UASTC_DICT_SIZE_MAX] // UASTC LDR 4x4 .function("setRDOUASTCDictSize", optional_override([](basis_encoder& self, int dict_size) { @@ -2641,6 +3027,65 @@ EMSCRIPTEN_BINDINGS(basis_codec) { self.m_params.m_rdo_uastc_ldr_4x4_skip_block_rms_thresh = rdo_uastc_skip_block_rms_thresh; })) + // XUASTC/ASTC LDR 4x4-12x12 specific options + + // Enable XUASTC LDR DCT usage. Recommended to also enabled lossy supercompression for more compression. + // DCT quality [1,100] is set via setQualityLevel() above. + .function("setXUASTCLDRUseDCT", optional_override([](basis_encoder& self, bool xuastc_use_dct) { + self.m_params.m_xuastc_ldr_use_dct = xuastc_use_dct; + })) + + // Enables lossy XUASTC LDR supercompression (bounded distortion/windowed RDO) + .function("setXUASTCLDRUseLossySupercompression", optional_override([](basis_encoder& self, bool xuastc_use_lossy_supercompression) { + self.m_params.m_xuastc_ldr_use_lossy_supercompression = xuastc_use_lossy_supercompression; + })) + + // XUASTC LDR: Disable 2-3 subset usage, independent of effort level (for lower quality, for faster transcoding to BC7) + .function("setXUASTCLDRForceDisableSubsets", optional_override([](basis_encoder& self, bool flag) { + self.m_params.m_xuastc_ldr_force_disable_subsets = flag; + })) + + // XUASTC LDR: Disable RGB dual plane usage, indepdnent of effort level (for lower quality, for faster transcoding to BC7) + .function("setXUASTCLDRForceDisableRGBDualPlane", optional_override([](basis_encoder& self, bool flag) { + self.m_params.m_xuastc_ldr_force_disable_rgb_dual_plane = flag; + })) + + // Sets the XUASTC LDR syntax: see the xuastc_ldr_syntax enum. + .function("setXUASTCLDRSyntax", optional_override([](basis_encoder& self, int syntax) { + self.m_params.m_xuastc_ldr_syntax = syntax; + })) + + // Sets the ASTC/XUASTC LDR: compressor effort level [0,10] (encoding time vs. max achievable quality tradeoff, higher=slower) + // This is like setCompressionLevel() above, but for only ASTC/UASTC LDR 4x4-12x12, and has a different range. + .function("setASTCOrXUASTCLDREffortLevel", optional_override([](basis_encoder& self, int effort_level) { + self.m_params.m_xuastc_ldr_effort_level = effort_level; + })) + + // Sets the ASTC/XUASTC LDR channel weights + .function("setASTCOrXUASTCLDRWeights", optional_override([](basis_encoder& self, uint32_t x, uint32_t y, uint32_t z, uint32_t w) { + self.m_params.m_xuastc_ldr_channel_weights[0] = x; + self.m_params.m_xuastc_ldr_channel_weights[1] = y; + self.m_params.m_xuastc_ldr_channel_weights[2] = z; + self.m_params.m_xuastc_ldr_channel_weights[3] = w; + })) + + // Sets XUASTC LDR lossy supercompression (bounded/windows RDO) parameters. + // Must be enabled via setXUASTCLDRUseLossySupercompression(). + .function("setXUASTCLDRBoundedRDOParam", optional_override([](basis_encoder& self, uint32_t idx, float value) { + switch (idx) + { + case 0: self.m_params.m_ls_min_psnr = value; break; + case 1: self.m_params.m_ls_min_alpha_psnr = value; break; + case 2: self.m_params.m_ls_thresh_psnr = value; break; + case 3: self.m_params.m_ls_thresh_alpha_psnr = value; break; + case 4: self.m_params.m_ls_thresh_edge_psnr = value; break; + case 5: self.m_params.m_ls_thresh_edge_alpha_psnr = value; break; + default: + assert(0); + break; + } + })) + // --- Low level options // Disables ETC1S selector RDO diff --git a/webgl/webserver_cross_origin.py b/webgl/webserver_cross_origin.py index 8af0eda..d251101 100644 --- a/webgl/webserver_cross_origin.py +++ b/webgl/webserver_cross_origin.py @@ -7,7 +7,7 @@ class CORSRequestHandler(http.server.SimpleHTTPRequestHandler): self.send_header("Cross-Origin-Embedder-Policy", "require-corp") super().end_headers() -PORT = 8080 +PORT = 8081 with socketserver.TCPServer(("", PORT), CORSRequestHandler) as httpd: print(f"Serving at http://localhost:{PORT}") httpd.serve_forever() diff --git a/zstd/zstd.c b/zstd/zstd.c index d47e6dd..43958d8 100644 --- a/zstd/zstd.c +++ b/zstd/zstd.c @@ -4,11 +4,11 @@ * * Generate using: * \code - * combine.sh -r ../../lib -o zstd.c zstd-in.c + * python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstd.c zstd-in.c * \endcode */ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -25,9 +25,13 @@ * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also * defined in mem.h (breaking C99 compatibility). * - * Note: the undefs for xxHash allow Zstd's implementation to coinside with with + * Note: the undefs for xxHash allow Zstd's implementation to coincide with * standalone xxHash usage (with global defines). * + * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need + * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the + * legacy support at the source level). + * * Note: multithreading is enabled for all platforms apart from Emscripten. */ #define DEBUGLEVEL 0 @@ -43,13 +47,15 @@ #define ZSTD_MULTITHREAD #endif #define ZSTD_TRACE 0 +/* TODO: Can't amalgamate ASM function */ +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC #define ZSTD_DEPS_NEED_MATH64 /**** start inlining common/zstd_deps.h ****/ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -74,6 +80,18 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON +/* Even though we use qsort_r only for the dictionary builder, the macro + * _GNU_SOURCE has to be declared *before* the inclusion of any standard + * header and the script 'combine.sh' combines the whole zstd source code + * in a single file. + */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include #include #include @@ -165,7 +183,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -187,7 +205,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -218,10 +236,6 @@ #ifndef DEBUG_H_12987983217 #define DEBUG_H_12987983217 -#if defined (__cplusplus) -extern "C" { -#endif - /* static assert is triggered at compile time, leaving no runtime artefact. * static assert only works with compile-time constants. @@ -271,34 +285,44 @@ extern int g_debuglevel; /* the variable is only declared, It's useful when enabling very verbose levels on selective conditions (such as position in src) */ -# define RAWLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__VA_ARGS__); \ - } } -# define DEBUGLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ - ZSTD_DEBUG_PRINT(" \n"); \ - } } +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) + +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) #else -# define RAWLOG(l, ...) {} /* disabled */ -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -#if defined (__cplusplus) -} +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ #endif #endif /* DEBUG_H_12987983217 */ /**** ended inlining debug.h ****/ +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ int g_debuglevel = DEBUGLEVEL; +#endif /**** ended inlining common/debug.c ****/ /**** start inlining common/entropy_common.c ****/ /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -315,7 +339,7 @@ int g_debuglevel = DEBUGLEVEL; ***************************************/ /**** start inlining mem.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -327,17 +351,13 @@ int g_debuglevel = DEBUGLEVEL; #ifndef MEM_H_MODULE #define MEM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - /*-**************************************** * Dependencies ******************************************/ #include /* size_t, ptrdiff_t */ /**** start inlining compiler.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -349,6 +369,182 @@ extern "C" { #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include + +/**** start inlining portability_macros.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Compile time determination of BMI2 support */ +#ifndef STATIC_BMI2 +# if defined(__BMI2__) +# define STATIC_BMI2 1 +# elif defined(_MSC_VER) && defined(__AVX2__) +# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */ +# endif +#endif + +#ifndef STATIC_BMI2 +# define STATIC_BMI2 0 +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 +# if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \ + && !defined(__BMI2__) +# define DYNAMIC_BMI2 1 +# else +# define DYNAMIC_BMI2 0 +# endif +#endif + +/** + * Only enable assembly for GNU C compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS / Win32, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ +/**** ended inlining portability_macros.h ****/ + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -361,7 +557,7 @@ extern "C" { # define INLINE_KEYWORD #endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -378,7 +574,7 @@ extern "C" { /** On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). - This explictly marks such functions as __cdecl so that the code will still compile + This explicitly marks such functions as __cdecl so that the code will still compile if a CC other than __cdecl has been made the default. */ #if defined(_MSC_VER) @@ -387,12 +583,19 @@ extern "C" { # define WIN_CDECL #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR /** * HINT_INLINE is used to help the compiler generate better code. It is *not* * used for "templates", so it can be tweaked based on the compilers @@ -407,21 +610,37 @@ extern "C" { #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 # define HINT_INLINE static INLINE_KEYWORD #else -# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +# define HINT_INLINE FORCE_INLINE_TEMPLATE #endif -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ #if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined(__IAR_SYSTEMS_ICC__) +# define MEM_STATIC static inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline #else -# define UNUSED_ATTR +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif #endif /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# if defined(__GNUC__) || defined(__ICCARM__) +# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -430,68 +649,56 @@ extern "C" { /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) #endif -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X86)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ #else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ -//rg: fix for ARM64EC compilation -//# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) # else -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ # endif #endif /* NO_PREFETCH */ #define CACHELINE_SIZE 64 -#define PREFETCH_AREA(p, s) { \ - const char* const _ptr = (const char*)(p); \ - size_t const _size = (size_t)(s); \ - size_t _pos; \ - for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH_L2(_ptr + _pos); \ - } \ -} +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) /* vectorization - * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ -#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) # else @@ -514,6 +721,12 @@ extern "C" { #define UNLIKELY(x) (x) #endif +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ @@ -524,39 +737,198 @@ extern "C" { # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #endif -/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ -#ifndef STATIC_BMI2 -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) -# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 -# define STATIC_BMI2 1 +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__AVX2__) +# define ZSTD_ARCH_X86_AVX2 +# endif +# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_AVX2) +# include +# endif +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment +*****************************************************************/ + +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +#ifndef ZSTD_ALIGNED +/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */ +# if defined(__GNUC__) || defined(__clang__) +# define ZSTD_ALIGNED(a) __attribute__((aligned(a))) +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define ZSTD_ALIGNED(a) _Alignas(a) +#elif defined(_MSC_VER) +# define ZSTD_ALIGNED(n) __declspec(align(n)) +# else + /* this compiler will require its own alignment instruction */ +# define ZSTD_ALIGNED(...) +# endif +#endif /* ZSTD_ALIGNED */ + + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) # endif -# endif -#endif - -#ifndef STATIC_BMI2 - #define STATIC_BMI2 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 -#endif - -/* detects whether we are being compiled under msan */ -#ifndef ZSTD_MEMORY_SANITIZER -# if __has_feature(memory_sanitizer) -# define ZSTD_MEMORY_SANITIZER 1 # else -# define ZSTD_MEMORY_SANITIZER 0 +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR # endif #endif -#if ZSTD_MEMORY_SANITIZER +/** + * Helper function to perform a wrapped pointer difference without triggering + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) /* Not all platforms that support msan provide sanitizers/msan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -575,20 +947,13 @@ void __msan_poison(const volatile void *a, size_t size); /* Returns the offset of the first (at least partially) poisoned byte in the memory range, or -1 if the whole range is good. */ intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); #endif -/* detects whether we are being compiled under asan */ -#ifndef ZSTD_ADDRESS_SANITIZER -# if __has_feature(address_sanitizer) -# define ZSTD_ADDRESS_SANITIZER 1 -# elif defined(__SANITIZE_ADDRESS__) -# define ZSTD_ADDRESS_SANITIZER 1 -# else -# define ZSTD_ADDRESS_SANITIZER 0 -# endif -#endif - -#if ZSTD_ADDRESS_SANITIZER +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -638,15 +1003,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #if defined(_MSC_VER) /* Visual Studio */ # include /* _byteswap_ulong */ # include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#elif defined(__ICCARM__) +# include #endif /*-************************************************************** @@ -659,6 +1017,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # include /* intptr_t */ # endif typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; typedef uint16_t U16; typedef int16_t S16; typedef uint32_t U32; @@ -671,6 +1031,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # error "this implementation requires char to be exactly 8-bit type" #endif typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; #if USHRT_MAX != 65535 # error "this implementation requires short to be exactly 16-bit type" #endif @@ -687,7 +1049,6 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); typedef signed long long S64; #endif - /*-************************************************************** * Memory I/O API *****************************************************************/ @@ -737,23 +1098,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -763,8 +1116,24 @@ MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_X64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__ + return 1; +#else const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; +#endif } #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) @@ -782,30 +1151,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -849,6 +1207,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -856,14 +1222,25 @@ MEM_STATIC U32 MEM_swap32(U32 in) #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); +#elif defined(__ICCARM__) + return __REV(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + MEM_STATIC U64 MEM_swap64(U64 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -872,14 +1249,7 @@ MEM_STATIC U64 MEM_swap64(U64 in) || (defined(__clang__) && __has_builtin(__builtin_bswap64)) return __builtin_bswap64(in); #else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); + return MEM_swap64_fallback(in); #endif } @@ -916,7 +1286,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) MEM_STATIC U32 MEM_readLE24(const void* memPtr) { - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); } MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) @@ -1026,16 +1396,11 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) /* code only tested on 32 and 64 bits systems */ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - -#if defined (__cplusplus) -} -#endif - #endif /* MEM_H_MODULE */ /**** ended inlining mem.h ****/ /**** start inlining error_private.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1049,18 +1414,12 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si #ifndef ERROR_H_MODULE #define ERROR_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - - /* **************************************** * Dependencies ******************************************/ -/**** skipping file: zstd_deps.h ****/ -/**** start inlining zstd_errors.h ****/ +/**** start inlining ../zstd_errors.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1076,24 +1435,32 @@ extern "C" { extern "C" { #endif -/*===== dependency =====*/ -#include /* size_t */ - - /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDERRORLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE # endif #endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE #endif /*-********************************************* @@ -1119,14 +1486,18 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -1134,18 +1505,18 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare with enum list published above */ -ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ @@ -1154,8 +1525,10 @@ ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Sa #endif #endif /* ZSTD_ERRORS_H_398273423 */ -/**** ended inlining zstd_errors.h ****/ - +/**** ended inlining ../zstd_errors.h ****/ +/**** skipping file: compiler.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: zstd_deps.h ****/ /* **************************************** * Compiler-specific @@ -1190,8 +1563,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /* check and forward error code */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) /*-**************************************** @@ -1205,9 +1583,86 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) return ERR_getErrorString(ERR_getErrorCode(code)); } -#if defined (__cplusplus) +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; } -#endif + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) #endif /* ERROR_H_MODULE */ /**** ended inlining error_private.h ****/ @@ -1216,7 +1671,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1226,11 +1681,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ - -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef FSE_H #define FSE_H @@ -1240,7 +1690,6 @@ extern "C" { ******************************************/ /**** skipping file: zstd_deps.h ****/ - /*-***************************************** * FSE_PUBLIC_API : control library symbols visibility ******************************************/ @@ -1268,34 +1717,6 @@ extern "C" { FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ @@ -1306,20 +1727,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -1379,8 +1786,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -1456,23 +1861,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -1504,15 +1893,14 @@ If there is an error, the function will return an error code, which can be teste #endif /* FSE_H */ + #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) #define FSE_H_FSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ /**** start inlining bitstream.h ****/ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1525,9 +1913,6 @@ If there is an error, the function will return an error code, which can be teste #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -1541,14 +1926,220 @@ extern "C" { /**** skipping file: compiler.h ****/ /**** skipping file: debug.h ****/ /**** skipping file: error_private.h ****/ +/**** start inlining bits.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +/**** skipping file: mem.h ****/ + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctz(val); +#else + return ZSTD_countTrailingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_clz(val); +#else + return ZSTD_countLeadingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctzll(val); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +#elif defined(__ICCARM__) + return (unsigned)(__builtin_clzll(val)); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ +/**** ended inlining bits.h ****/ /*========================================= * Target specific =========================================*/ #ifndef ZSTD_NO_INTRINSICS -# if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ # elif defined(__ICCARM__) # include # endif @@ -1562,12 +2153,13 @@ extern "C" { /*-****************************************** * bitStream encoding API (write forward) ********************************************/ +typedef size_t BitContainerType; /* bitStream can mix input from multiple sources. * A critical property of these streams is that they encode and decode in **reverse** direction. * So the first bit sequence you add will be the last to be read, like a LIFO stack. */ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitPos; char* startPtr; char* ptr; @@ -1575,7 +2167,7 @@ typedef struct { } BIT_CStream_t; MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); @@ -1584,7 +2176,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems. * Writing data into memory is an explicit operation, performed by the flushBits function. * Hence keep track how many bits are potentially stored into local register to avoid register overflow. * After a flushBits, a maximum of 7 bits might still be stored into local register. @@ -1601,28 +2193,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * bitStream decoding API (read backward) **********************************************/ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitsConsumed; const char* ptr; const char* start; const char* limitPtr; } BIT_DStream_t; -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /* Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. @@ -1634,7 +2226,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /*-**************************************** * unsafe API ******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); @@ -1643,42 +2235,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - unsigned long r = 0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -1708,16 +2264,31 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits) +{ +#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS) +# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__) + return _bzhi_u64(bitContainer, nbBits); +# else + DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32)); + return _bzhi_u32(bitContainer, nbBits); +# endif +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -1725,7 +2296,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, * works only if `value` is _clean_, * meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { assert((value>>nbBits) == 0); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); @@ -1772,7 +2343,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_flushBits(bitC); if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); + return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); } @@ -1796,35 +2367,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - /* fall-through */ + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - /* fall-through */ + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - /* fall-through */ + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - /* fall-through */ + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - /* fall-through */ + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - /* fall-through */ + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -1833,26 +2404,25 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ assert(nbBits < BIT_MASK_SIZE); - return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; -} - -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X64) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); #else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; #endif } @@ -1862,7 +2432,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -1878,14 +2448,14 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U3 /*! BIT_lookBitsFast() : * unsafe version; only works if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) { U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; assert(nbBits >= 1); return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -1894,23 +2464,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBits(bitD, nbBits); + BitContainerType const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); return value; } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBitsFast(bitD, nbBits); + BitContainerType const value = BIT_lookBitsFast(bitD, nbBits); assert(nbBits >= 1); BIT_skipBits(bitD, nbBits); return value; } +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStreamFast() : * Similar to BIT_reloadDStream(), but with two differences: * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! @@ -1921,31 +2506,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) { if (UNLIKELY(bitD->ptr < bitD->limitPtr)) return BIT_DStream_overflow; - assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStream_internal(bitD); } /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . - * This function is safe, it guarantees it will not read beyond src buffer. + * This function is safe, it guarantees it will not never beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); if (bitD->ptr >= bitD->limitPtr) { - return BIT_reloadDStreamFast(bitD); + return BIT_reloadDStream_internal(bitD); } if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } - /* start < ptr < limitPtr */ + /* start < ptr < limitPtr => cautious update */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -1967,14 +2556,9 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); } -#if defined (__cplusplus) -} -#endif - #endif /* BITSTREAM_H_MODULE */ /**** ended inlining bitstream.h ****/ - /* ***************************************** * Static allocation *******************************************/ @@ -1999,24 +2583,15 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. */ -#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); @@ -2025,19 +2600,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -2220,20 +2787,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); + BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) { - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog); BIT_flushBits(bitC); } /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) @@ -2386,20 +2953,13 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) - #endif /* FSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif /**** ended inlining fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ /**** start inlining huf.h ****/ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -2410,115 +2970,33 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef HUF_H_298734234 #define HUF_H_298734234 /* *** Dependencies *** */ /**** skipping file: zstd_deps.h ****/ - - -/* *** library symbols visibility *** */ -/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, - * HUF symbols remain "private" (internal symbols for library only). - * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ -#else -# define HUF_PUBLIC_API -#endif - - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) -#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); - -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ /**** skipping file: mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) /* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ #define HUF_SYMBOLVALUE_MAX 255 -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) # error "HUF_TABLELOG_MAX is too large !" #endif @@ -2534,15 +3012,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, /* static allocation of HUF's Compression Table */ /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt */ -typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ -#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ -#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ /* static allocation of HUF's DTable */ typedef U32 HUF_DTable; @@ -2556,25 +3030,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -2587,10 +3085,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); @@ -2599,22 +3099,24 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, @@ -2640,17 +3142,29 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); -/** HUF_getNbBits() : +/** HUF_getNbBitsFromCTable() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX - * Note 1 : is not inlined, as HUF_CElt definition is private - * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ -U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); /* * HUF_decompress() does the following: @@ -2676,88 +3190,51 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); * a required workspace size greater than that specified in the following * macro. */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif -#endif /* HUF_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} -#endif +#endif /* HUF_H_298734234 */ /**** ended inlining huf.h ****/ +/**** skipping file: bits.h ****/ /*=== Version ===*/ @@ -2775,28 +3252,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanForward(&r, val) ? (unsigned)r : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -2844,7 +3299,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -2855,7 +3310,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -2920,7 +3375,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -2954,7 +3409,7 @@ static size_t FSE_readNCount_body_default( } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -2995,7 +3450,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -3036,21 +3491,21 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected); rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -3074,7 +3529,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) @@ -3087,20 +3542,20 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } /**** ended inlining common/entropy_common.c ****/ /**** start inlining common/error_private.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3128,9 +3583,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -3139,17 +3596,23 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } @@ -3159,7 +3622,7 @@ const char* ERR_getErrorString(ERR_enum code) /**** start inlining common/fse_decompress.c ****/ /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -3181,8 +3644,8 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ /**** skipping file: error_private.h ****/ -#define ZSTD_DEPS_NEED_MALLOC /**** skipping file: zstd_deps.h ****/ +/**** skipping file: bits.h ****/ /* ************************************************************** @@ -3214,19 +3677,6 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -3255,7 +3705,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + symbolNext[s] = (U16)normalizedCounter[s]; } } } ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -3270,8 +3720,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo * all symbols have counts <= 8. We ensure we have 8 bytes at the end of * our buffer to handle the over-write. */ - { - U64 const add = 0x0101010101010101ull; + { U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0; U32 s; @@ -3282,14 +3731,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; - } - } + pos += (size_t)n; + } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -3325,7 +3773,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s= ostart); + return (size_t)(op-ostart); } +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); -} FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( void* dst, size_t dstCapacity, @@ -3477,24 +3870,34 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - FSE_DTable* const dtable = (FSE_DTable*)workSpace; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); - if (FSE_isError(NCountLength)) return NCountLength; - if (tableLog > maxLog) return ERROR(tableLog_tooLarge); - assert(NCountLength <= cSrcSize); - ip += NCountLength; - cSrcSize -= NCountLength; + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog); - wkspSize -= FSE_DTABLE_SIZE(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); - CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) ); + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); { const void* ptr = dtable; @@ -3514,7 +3917,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } @@ -3531,24 +3934,6 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; - return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); -} - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - /* Static analyzer seems unable to understand this table will be properly initialized later */ - U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); -} -#endif - - #endif /* FSE_COMMONDEFS_ONLY */ /**** ended inlining common/fse_decompress.c ****/ /**** start inlining common/threading.c ****/ @@ -3588,15 +3973,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS /**** skipping file: debug.h ****/ -#if defined (__cplusplus) -extern "C" { -#endif - #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ #ifdef WINVER # undef WINVER @@ -3634,22 +4014,17 @@ extern "C" { #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) /* ZSTD_pthread_create() and ZSTD_pthread_join() */ -typedef struct { - HANDLE handle; - void* (*start_routine)(void*); - void* arg; -} ZSTD_pthread_t; +typedef HANDLE ZSTD_pthread_t; int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg); -int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); +int ZSTD_pthread_join(ZSTD_pthread_t thread); /** * add here more wrappers as required */ - #elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ /* === POSIX Systems === */ # include @@ -3671,7 +4046,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #else /* DEBUGLEVEL >= 1 */ @@ -3696,7 +4071,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #endif @@ -3720,9 +4095,6 @@ typedef int ZSTD_pthread_cond_t; #endif /* ZSTD_MULTITHREAD */ -#if defined (__cplusplus) -} -#endif #endif /* THREADING_H_938743 */ /**** ended inlining threading.h ****/ @@ -3733,8 +4105,7 @@ int g_ZSTD_threading_useless_symbol; #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ @@ -3745,37 +4116,94 @@ int g_ZSTD_threading_useless_symbol; /* === Implementation === */ +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + static unsigned __stdcall worker(void *arg) { - ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; - thread->arg = thread->start_routine(thread->arg); + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + return 0; } int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg) { + ZSTD_thread_params_t thread_param; (void)unused; - thread->arg = arg; - thread->start_routine = start_routine; - thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); - if (!thread->handle) + if (thread==NULL) return -1; + *thread = NULL; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (*thread==NULL) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); return errno; - else - return 0; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; } -int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) +int ZSTD_pthread_join(ZSTD_pthread_t thread) { DWORD result; - if (!thread.handle) return 0; + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); - result = WaitForSingleObject(thread.handle, INFINITE); switch (result) { case WAIT_OBJECT_0: - if (value_ptr) *value_ptr = thread.arg; return 0; case WAIT_ABANDONED: return EINVAL; @@ -3793,6 +4221,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) { + assert(mutex != NULL); *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); if (!*mutex) return 1; @@ -3801,6 +4230,7 @@ int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t con int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) { + assert(mutex != NULL); if (!*mutex) return 0; { @@ -3812,6 +4242,7 @@ int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) { + assert(cond != NULL); *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); if (!*cond) return 1; @@ -3820,6 +4251,7 @@ int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) { + assert(cond != NULL); if (!*cond) return 0; { @@ -3833,7 +4265,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) /**** ended inlining common/threading.c ****/ /**** start inlining common/pool.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3844,11 +4276,9 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) /* ====== Dependencies ======= */ -/**** skipping file: zstd_deps.h ****/ -/**** skipping file: debug.h ****/ -/**** start inlining zstd_internal.h ****/ +/**** start inlining ../common/allocations.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3857,4042 +4287,58 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) * You may select, at your option, one of the above-listed licenses. */ -#ifndef ZSTD_CCOMMON_H_MODULE -#define ZSTD_CCOMMON_H_MODULE - -/* this module contains definitions which must be identical - * across compression, decompression and dictBuilder. - * It also contains a few functions useful to at least 2 of them - * and which benefit from being inlined */ - -/*-************************************* -* Dependencies -***************************************/ -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) -#include -#endif -/**** skipping file: compiler.h ****/ -/**** skipping file: mem.h ****/ -/**** skipping file: debug.h ****/ -/**** skipping file: error_private.h ****/ -#define ZSTD_STATIC_LINKING_ONLY -/**** start inlining ../zstd.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef ZSTD_H_235446 -#define ZSTD_H_235446 - -/* ====== Dependency ======*/ -#include /* INT_MAX */ -#include /* size_t */ - - -/* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define ZSTDLIB_VISIBILITY -# endif -#endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY -#endif - - -/******************************************************************************* - Introduction - - zstd, short for Zstandard, is a fast lossless compression algorithm, targeting - real-time compression scenarios at zlib-level and better compression ratios. - The zstd compression library provides in-memory compression and decompression - functions. - - The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), - which is currently 22. Levels >= 20, labeled `--ultra`, should be used with - caution, as they require more memory. The library also offers negative - compression levels, which extend the range of speed vs. ratio preferences. - The lower the level, the faster the speed (at the cost of compression). - - Compression can be done in: - - a single step (described as Simple API) - - a single step, reusing a context (described as Explicit context) - - unbounded multiple steps (described as Streaming compression) - - The compression ratio achievable on small data can be highly improved using - a dictionary. Dictionary compression can be performed in: - - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Bulk-processing - dictionary API) - - Advanced experimental functions can be accessed using - `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. - - Advanced experimental APIs should never be used with a dynamically-linked - library. They are not "stable"; their definitions or signatures may change in - the future. Only static linking is allowed. -*******************************************************************************/ - -/*------ Version ------*/ -#define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) - -/*! ZSTD_versionNumber() : - * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ -ZSTDLIB_API unsigned ZSTD_versionNumber(void); - -#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE -#define ZSTD_QUOTE(str) #str -#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) -#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) - -/*! ZSTD_versionString() : - * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ -ZSTDLIB_API const char* ZSTD_versionString(void); - -/* ************************************* - * Default constant - ***************************************/ -#ifndef ZSTD_CLEVEL_DEFAULT -# define ZSTD_CLEVEL_DEFAULT 3 -#endif - -/* ************************************* - * Constants - ***************************************/ - -/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ -#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ -#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 - -#define ZSTD_BLOCKSIZELOG_MAX 17 -#define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) -ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); - -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). - * Both functions work the same way, but ZSTD_getDecompressedSize() blends - * "empty", "unknown" and "error" results to the same return value (0), - * while ZSTD_getFrameContentSize() gives them separate return values. - * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_findFrameCompressedSize() : - * `src` should point to the start of a ZSTD frame or skippable frame. - * `srcSize` must be >= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); - - -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ - - -/*************************************** -* Explicit context -***************************************/ -/*= Compression context - * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Note : re-using context is just a speed / resource optimization. - * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. - */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); - -/*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . - * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. - */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*= Decompression context - * When decompressing many times, - * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution. */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); - -/*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), - * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. - */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/*************************************** -* Advanced compression API -***************************************/ - -/* API design : - * Parameters are pushed one by one into an existing context, - * using ZSTD_CCtx_set*() functions. - * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. - * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . - * - * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). - * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. - */ - - -/* Compression strategies, listed from fastest to strongest */ -typedef enum { ZSTD_fast=1, - ZSTD_dfast=2, - ZSTD_greedy=3, - ZSTD_lazy=4, - ZSTD_lazy2=5, - ZSTD_btlazy2=6, - ZSTD_btopt=7, - ZSTD_btultra=8, - ZSTD_btultra2=9 - /* note : new strategies _might_ be added in the future. - Only the order (from fast to strong) is guaranteed */ -} ZSTD_strategy; - - -typedef enum { - - /* compression parameters - * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. - * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. - * Note that exact compression parameters are dynamically determined, - * depending on both compression level and srcSize (when known). - * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. - * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level does not automatically set all other compression parameters - * to default. Setting this will however eventually dynamically impact the compression - * parameters which have not been manually set. The manually set - * ones will 'stick'. */ - /* Advanced compression parameters : - * It's possible to pin down compression parameters to some specific values. - * In which case, these values are no longer dynamically selected by the compressor */ - ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. - * This will set a memory budget for streaming decompression, - * with larger values requiring more memory - * and typically compressing more. - * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "use default windowLog". - * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such size at streaming decompression stage. */ - ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. - * Resulting memory usage is (1 << (hashLog+2)). - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. - * Larger tables improve compression ratio of strategies <= dFast, - * and improve speed of strategies > dFast. - * Special: value 0 means "use default hashLog". */ - ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. - * Resulting memory usage is (1 << (chainLog+2)). - * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. - * Larger tables result in better and slower compression. - * This parameter is useless for "fast" strategy. - * It's still useful when using "dfast" strategy, - * in which case it defines a secondary probe table. - * Special: value 0 means "use default chainLog". */ - ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. - * More attempts result in better and slower compression. - * This parameter is useless for "fast" and "dFast" strategies. - * Special: value 0 means "use default searchLog". */ - ZSTD_c_minMatch=105, /* Minimum size of searched matches. - * Note that Zstandard can still find matches of smaller size, - * it just tweaks its search algorithm to look for this size and larger. - * Larger values increase compression and decompression speed, but decrease ratio. - * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. - * Note that currently, for all strategies < btopt, effective minimum is 4. - * , for all strategies > fast, effective maximum is 6. - * Special: value 0 means "use default minMatchLength". */ - ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. - * For strategies btopt, btultra & btultra2: - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger, and slower. - * For strategy fast: - * Distance between match sampling. - * Larger values make compression faster, and weaker. - * Special: value 0 means "use default targetLength". */ - ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. - * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. - * Special: value 0 means "use default strategy". */ - - /* LDM mode parameters */ - ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. - * This parameter is designed to improve compression ratio - * for large inputs, by finding large matches at long distance. - * It increases memory usage and window size. - * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. - * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and - * compression strategy >= ZSTD_btopt (== compression level 16+) */ - ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, - * but decrease compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * default: windowlog - 7. - * Special: value 0 means "automatically determine hashlog". */ - ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. - * Special: value 0 means "use default value" (default: 64). */ - ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. - * Special: value 0 means "use default value" (default: 3). */ - ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. - * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). - * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. - * Larger values improve compression speed. - * Deviating far from default value will likely result in a compression ratio decrease. - * Special: value 0 means "automatically determine hashRateLog". */ - - /* frame parameters */ - ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression. - * This is automatically the case when using ZSTD_compress2(), - * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ - ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ - ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ - - /* multi-threading parameters */ - /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). - * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. - * In a situation where it's unknown if the linked library supports multi-threading or not, - * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. - */ - ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : - * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression is performed in parallel, within worker thread(s). - * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : - * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). - * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, - * compression is performed inside Caller's thread, and all invocations are blocking */ - ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. - * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. - * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced. */ - ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. - * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. - * It helps preserve compression ratio, while each job is compressed in parallel. - * This value is enforced only when nbWorkers >= 1. - * Larger values increase compression ratio, but decrease speed. - * Possible values range from 0 to 9 : - * - 0 means "default" : value will be determined by the library, depending on strategy - * - 1 means "no overlap" - * - 9 means "full overlap", using a full window size. - * Each intermediate rank increases/decreases load size by a factor 2 : - * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default - * default value varies between 6 and 9, depending on strategy */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_c_rsyncable - * ZSTD_c_format - * ZSTD_c_forceMaxWindow - * ZSTD_c_forceAttachDict - * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize - * ZSTD_c_srcSizeHint - * ZSTD_c_enableDedicatedDictSearch - * ZSTD_c_stableInBuffer - * ZSTD_c_stableOutBuffer - * ZSTD_c_blockDelimiters - * ZSTD_c_validateSequences - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly; - * also, the enums values themselves are unstable and can still change. - */ - ZSTD_c_experimentalParam1=500, - ZSTD_c_experimentalParam2=10, - ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001, - ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, - ZSTD_c_experimentalParam8=1005, - ZSTD_c_experimentalParam9=1006, - ZSTD_c_experimentalParam10=1007, - ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 -} ZSTD_cParameter; - -typedef struct { - size_t error; - int lowerBound; - int upperBound; -} ZSTD_bounds; - -/*! ZSTD_cParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - lower and upper bounds, both inclusive - */ -ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); - -/*! ZSTD_CCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_cParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is generally only possible during frame initialization (before starting compression). - * Exception : when using multi-threading mode (nbWorkers >= 1), - * the following parameters can be updated _during_ compression (within same frame): - * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. - * new parameters will be active for next job only (after a flush()). - * @return : an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtx_setPledgedSrcSize() : - * Total input data size to be compressed as a single frame. - * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. - * This value will also be controlled at end of frame, and trigger an error if not respected. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. - * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. - * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. - * Note 2 : pledgedSrcSize is only valid once, for the next frame. - * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. - * Note 3 : Whenever all input data is provided and consumed in a single round, - * for example with ZSTD_compress2(), - * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - * this value is automatically overridden by srcSize instead. - */ -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); - -typedef enum { - ZSTD_reset_session_only = 1, - ZSTD_reset_parameters = 2, - ZSTD_reset_session_and_parameters = 3 -} ZSTD_ResetDirective; - -/*! ZSTD_CCtx_reset() : - * There are 2 different things that can be reset, independently or jointly : - * - The session : will stop compressing current frame, and make CCtx ready to start a new one. - * Useful after an error, or to interrupt any ongoing compression. - * Any internal data not yet flushed is cancelled. - * Compression parameters and dictionary remain unchanged. - * They will be used to compress next frame. - * Resetting session never fails. - * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. - * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) - * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) - * - Both : similar to resetting the session, followed by resetting parameters. - */ -ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); - -/*! ZSTD_compress2() : - * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. - * ZSTD_compress2() always starts a new frame. - * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/*************************************** -* Advanced decompression API -***************************************/ - -/* The advanced API pushes parameters one by one into an existing DCtx context. - * Parameters are sticky, and remain valid for all following frames - * using the same DCtx context. - * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). - * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). - * Therefore, no new decompression function is necessary. - */ - -typedef enum { - - ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which - * the streaming API will refuse to allocate memory buffer - * in order to protect the host from unreasonable memory requirements. - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). - * Special: value 0 means "use default maximum windowLog". */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_d_format - * ZSTD_d_stableOutBuffer - * ZSTD_d_forceIgnoreChecksum - * ZSTD_d_refMultipleDDicts - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly - */ - ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001, - ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 - -} ZSTD_dParameter; - -/*! ZSTD_dParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - both lower and upper bounds, inclusive - */ -ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); - -/*! ZSTD_DCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_dParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is only possible during frame initialization (before starting decompression). - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); - -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately. - * Parameters can only be reset when no active frame is being decompressed. - * @return : 0, or an error code, which can be tested with ZSTD_isError() - */ -ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); - - -/**************************** -* Streaming -****************************/ - -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - - -/*-*********************************************************************** -* Streaming compression - HowTo -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. -* -* For parallel execution, use one separate ZSTD_CStream per thread. -* -* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. -* -* Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. -* When in doubt, it's recommended to fully initialize the context before usage. -* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), -* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to -* set more specific parameters, the pledged source size, or load a dictionary. -* -* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to -* consume input stream. The function will automatically update both `pos` -* fields within `input` and `output`. -* Note that the function may not consume the entire input, for example, because -* the output buffer is already full, in which case `input.pos < input.size`. -* The caller must check if input has been entirely consumed. -* If not, the caller must make some room to receive more compressed data, -* and then present again remaining input data. -* note: ZSTD_e_continue is guaranteed to make some forward progress when called, -* but doesn't guarantee maximal forward progress. This is especially relevant -* when compressing with multiple threads. The call won't block if it can -* consume some input, but if it can't it will wait for some, but not all, -* output to be flushed. -* @return : provides a minimum amount of data remaining to be flushed from internal buffers -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, -* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. -* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). -* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the -* operation. -* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if internal buffers are entirely flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to -* start a new frame. -* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if frame fully completed and fully flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ - /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ -/*===== ZSTD_CStream management functions =====*/ -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -/*===== Streaming compression functions =====*/ -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush=1, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. - * note : multithreaded compression will block to flush as much output as possible. */ - ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. - * note that frame is only closed after compressed data is fully flushed (return value == 0). - * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). - : note : multithreaded compression will block to flush as much output as possible. */ -} ZSTD_EndDirective; - -/*! ZSTD_compressStream2() : - * Behaves about the same as ZSTD_compressStream, with additional control on end directive. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - * - output->pos must be <= dstCapacity, input->pos must be <= srcSize - * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - endOp must be a valid directive - * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/* These buffer sizes are softly recommended. - * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. - * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), - * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. - * - * However, note that these recommendations are from the perspective of a C caller program. - * If the streaming interface is invoked from some other language, - * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, - * a major performance rule is to reduce crossing such interface to an absolute minimum. - * It's not rare that performance ends being spent more into the interface, rather than compression itself. - * In which cases, prefer using large buffers, as large as practical, - * for both input and output, to reduce the nb of roundtrips. - */ -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ - - -/* ***************************************************************************** - * This following is a legacy streaming API. - * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). - * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. - ******************************************************************************/ - -/*! - * Equivalent to: - * - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - */ -ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/*! - * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). - * NOTE: The return value is different. ZSTD_compressStream() returns a hint for - * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the minimum nb of bytes left to flush (if non-zero and not an error). - */ -ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ -ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ -ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); - - -/*-*************************************************************************** -* Streaming decompression - HowTo -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation. -* @return : recommended first input size -* Alternatively, use advanced API to set specific properties. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos` fields. -* If `input.pos < input.size`, some input has not been consumed. -* It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, respecting output buffer size. -* If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., -* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. -* @return : 0 when a frame is completely decoded and fully flushed, -* or an error code, which can be tested using ZSTD_isError(), -* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. -* *******************************************************************************/ - -typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ - /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ -/*===== ZSTD_DStream management functions =====*/ -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -/*===== Streaming decompression functions =====*/ - -/* This function is redundant with the advanced API and equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, NULL); - */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); - -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - -ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ - - -/************************** -* Simple dictionary API -***************************/ -/*! ZSTD_compress_usingDict() : - * Compression at an explicit compression level using a Dictionary. - * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); - -/*! ZSTD_decompress_usingDict() : - * Decompression using a known Dictionary. - * Dictionary must be identical to the one used during compression. - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*********************************** - * Bulk processing dictionary API - **********************************/ -typedef struct ZSTD_CDict_s ZSTD_CDict; - -/*! ZSTD_createCDict() : - * When compressing multiple messages or blocks using the same dictionary, - * it's recommended to digest the dictionary only once, since it's a costly operation. - * ZSTD_createCDict() will create a state from digesting a dictionary. - * The resulting state can be used for future compression operations with very limited startup cost. - * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. - * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, - * in which case the only thing that it transports is the @compressionLevel. - * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, - * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); - -/*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); - -/*! ZSTD_compress_usingCDict() : - * Compression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. - * Note : compression level is _decided at dictionary creation time_, - * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); - - -typedef struct ZSTD_DDict_s ZSTD_DDict; - -/*! ZSTD_createDDict() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); - -/*! ZSTD_decompress_usingDDict() : - * Decompression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); - - -/******************************** - * Dictionary helper functions - *******************************/ - -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); - -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); - - -/******************************************************************************* - * Advanced dictionary and prefix API - * - * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. - ******************************************************************************/ - - -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - * Note 2 : Loading a dictionary involves building tables. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Tables are dependent on compression parameters, and for this reason, - * compression parameters can no longer be changed after loading a dictionary. - * Note 3 :`dict` content will be copied internally. - * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. - * Note that compression parameters are enforced from within CDict, - * and supersede any compression parameter previously set within CCtx. - * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. - * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. - * The dictionary will remain valid for future compressed frames using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Referencing a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); - -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compressed frame. - * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression. - * Its content must remain unmodified during compression. - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_c_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). - * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); - -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Loading a dictionary involves building tables, - * which has a non-negligible impact on CPU usage and latency. - * It's recommended to "load once, use many times", to amortize the cost - * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. - * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. - * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of - * how dictionary content is loaded and interpreted. - */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_refDDict() : - * Reference a prepared dictionary, to be used to decompress next frames. - * The dictionary remains active for decompression of future frames using same DCtx. - * - * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function - * will store the DDict references in a table, and the DDict used for decompression - * will be determined at decompression time, as per the dict ID in the frame. - * The memory for the table is allocated on the first call to refDDict, and can be - * freed with ZSTD_freeDCtx(). - * - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Special: referencing a NULL DDict means "return to no-dictionary mode". - * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -/*! ZSTD_DCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) to decompress next frame. - * This is the reverse operation of ZSTD_CCtx_refPrefix(), - * and must use the same prefix as the one used during compression. - * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_decompressStream() returns 0. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary - * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. - * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) - * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A full dictionary is more costly, as it requires building tables. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize); - -/* === Memory management === */ - -/*! ZSTD_sizeof_*() : - * These functions give the _current_ memory usage of selected object. - * Note that object memory usage can evolve (increase or decrease) over time. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); - -#endif /* ZSTD_H_235446 */ - - -/* ************************************************************************************** - * ADVANCED AND EXPERIMENTAL FUNCTIONS - **************************************************************************************** - * The definitions in the following section are considered experimental. - * They are provided for advanced scenarios. - * They should never be used with a dynamic library, as prototypes may change in the future. - * Use them only in association with static linking. - * ***************************************************************************************/ - -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY - -/**************************************************************************************** - * experimental API (static linking only) - **************************************************************************************** - * The following symbols and constants - * are not planned to join "stable API" status in the near future. - * They can still change in future versions. - * Some of them are planned to remain in the static_only section indefinitely. - * Some of them might be removed in the future (especially when redundant with existing stable functions) - * ***************************************************************************************/ - -#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ -#define ZSTD_SKIPPABLEHEADERSIZE 8 - -/* compression parameter bounds */ -#define ZSTD_WINDOWLOG_MAX_32 30 -#define ZSTD_WINDOWLOG_MAX_64 31 -#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX_32 29 -#define ZSTD_CHAINLOG_MAX_64 30 -#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ -#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX -#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ -#define ZSTD_STRATEGY_MIN ZSTD_fast -#define ZSTD_STRATEGY_MAX ZSTD_btultra2 - - -#define ZSTD_OVERLAPLOG_MIN 0 -#define ZSTD_OVERLAPLOG_MAX 9 - -#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame - * requiring larger than (1< 0: - * If litLength != 0: - * rep == 1 --> offset == repeat_offset_1 - * rep == 2 --> offset == repeat_offset_2 - * rep == 3 --> offset == repeat_offset_3 - * If litLength == 0: - * rep == 1 --> offset == repeat_offset_2 - * rep == 2 --> offset == repeat_offset_3 - * rep == 3 --> offset == repeat_offset_1 - 1 - * - * Note: This field is optional. ZSTD_generateSequences() will calculate the value of - * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not - * use this 'rep' field at all (as of now). - */ -} ZSTD_Sequence; - -typedef struct { - unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /**< dispatch table : larger == faster, more memory */ - unsigned searchLog; /**< nb of searches : larger == more compression, slower */ - unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ -} ZSTD_compressionParameters; - -typedef struct { - int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ - int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ - int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ -} ZSTD_frameParameters; - -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; - -typedef enum { - ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ - ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ - ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ -} ZSTD_dictContentType_e; - -typedef enum { - ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ -} ZSTD_dictLoadMethod_e; - -typedef enum { - ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. - * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring this instruction. */ -} ZSTD_format_e; - -typedef enum { - /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ - ZSTD_d_validateChecksum = 0, - ZSTD_d_ignoreChecksum = 1 -} ZSTD_forceIgnoreChecksum_e; - -typedef enum { - /* Note: this enum controls ZSTD_d_refMultipleDDicts */ - ZSTD_rmd_refSingleDDict = 0, - ZSTD_rmd_refMultipleDDicts = 1 -} ZSTD_refMultipleDDicts_e; - -typedef enum { - /* Note: this enum and the behavior it controls are effectively internal - * implementation details of the compressor. They are expected to continue - * to evolve and should be considered only in the context of extremely - * advanced performance tuning. - * - * Zstd currently supports the use of a CDict in three ways: - * - * - The contents of the CDict can be copied into the working context. This - * means that the compression can search both the dictionary and input - * while operating on a single set of internal tables. This makes - * the compression faster per-byte of input. However, the initial copy of - * the CDict's tables incurs a fixed cost at the beginning of the - * compression. For small compressions (< 8 KB), that copy can dominate - * the cost of the compression. - * - * - The CDict's tables can be used in-place. In this model, compression is - * slower per input byte, because the compressor has to search two sets of - * tables. However, this model incurs no start-up cost (as long as the - * working context's tables can be reused). For small inputs, this can be - * faster than copying the CDict's tables. - * - * - The CDict's tables are not used at all, and instead we use the working - * context alone to reload the dictionary and use params based on the source - * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). - * This method is effective when the dictionary sizes are very small relative - * to the input size, and the input size is fairly large to begin with. - * - * Zstd has a simple internal heuristic that selects which strategy to use - * at the beginning of a compression. However, if experimentation shows that - * Zstd is making poor choices, it is possible to override that choice with - * this enum. - */ - ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ - ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ -} ZSTD_dictAttachPref_e; - -typedef enum { - ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. - * Negative compression levels will be uncompressed, and positive compression - * levels will be compressed. */ - ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be - * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ -} ZSTD_literalCompressionMode_e; - - -/*************************************** -* Frame size functions -***************************************/ - -/*! ZSTD_findDecompressedSize() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - decompressed size of all data in all successive frames - * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * note 2 : decompressed size is always present when compression is done with ZSTD_compress() - * note 3 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure result fits within application's authorized limits. - * Each application can set its own limits. - * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - * read each contained frame header. This is fast as most of the data is skipped, - * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_decompressBound() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - upper-bound for the decompressed size of all data in all successive frames - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. - * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. - * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. - * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: - * upper-bound = # blocks * min(128 KB, Window_Size) - */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); - -/*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. - * @return : size of the Frame Header, - * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); - -typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; - -/*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. - * - * Each block will end with a dummy sequence - * with offset == 0, matchLength == 0, and litLength == length of last literals. - * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) - * simply acts as a block delimiter. - * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated - */ - -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); - -/*! ZSTD_mergeBlockDelimiters() : - * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. - * - * As such, the final generated result has no explicit representation of block boundaries, - * and the final last literals segment is not represented in the sequences. - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters - * @return : number of sequences left after merging - */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); - -/*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) - * The entire source is compressed into a single frame. - * - * The compression behavior changes based on cctx params. In particular: - * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on - * the block size derived from the cctx, and sequences may be split. This is the default setting. - * - * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. - * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. - * - * In addition to the two adjustable experimental params, there are other important cctx params. - * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. - * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. - * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset - * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md - * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); - - -/*! ZSTD_writeSkippableFrame() : - * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. - * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, - * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. - * - * Returns an error if destination buffer is not large enough, if the source size is not representable - * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). - * - * @return : number of bytes written or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); - - -/*************************************** -* Memory management -***************************************/ - -/*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * - * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. - * The estimate will assume the input may be arbitrarily large, - * which is the worst case. - * - * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions - * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), - * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). - * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. - * - * Note 2 : only single-threaded compression is supported. - * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. - */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); - -/*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. - * This information can be passed manually, using ZSTD_estimateDStreamSize, - * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); - * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), - * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); - -/*! ZSTD_estimate?DictSize() : - * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). - * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). - * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. - */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); - -/*! ZSTD_initStatic*() : - * Initialize an object using a pre-allocated fixed-size buffer. - * workspace: The memory area to emplace the object into. - * Provided pointer *must be 8-bytes aligned*. - * Buffer must outlive object. - * workspaceSize: Use ZSTD_estimate*Size() to determine - * how large workspace must be to support target scenario. - * @return : pointer to object (same address as workspace, just different type), - * or NULL if error (size too small, incorrect alignment, etc.) - * Note : zstd will never resize nor malloc() when using a static buffer. - * If the object requires more memory than available, - * zstd will just error out (typically ZSTD_error_memory_allocation). - * Note 2 : there is no corresponding "free" function. - * Since workspace is allocated externally, it must be freed externally too. - * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level - * into its associated cParams. - * Limitation 1 : currently not compatible with internal dictionary creation, triggered by - * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). - * Limitation 2 : static cctx currently not compatible with multi-threading. - * Limitation 3 : static dctx is incompatible with legacy support. - */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ - -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ - -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams); - -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType); - - -/*! Custom memory allocation : - * These prototypes make it possible to pass your own allocation/free functions. - * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. - */ -typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTD_freeFunction) (void* opaque, void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams, - ZSTD_customMem customMem); - -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. - */ -typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); - - -/* - * This API is temporary and is expected to change or disappear in the future! - */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - const ZSTD_CCtx_params* cctxParams, - ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem); - - -/*************************************** -* Advanced compression functions -***************************************/ - -/*! ZSTD_createCDict_byReference() : - * Create a digested dictionary for compression - * Dictionary content is just referenced, not duplicated. - * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. - * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); - -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); - -/*! ZSTD_getCParams() : - * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. - * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_getParams() : - * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. - * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_checkCParams() : - * Ensure param values remain within authorized range. - * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); - -/*! ZSTD_adjustCParams() : - * optimize params for a given `srcSize` and `dictSize`. - * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. - * `dictSize` must be `0` when there is no dictionary. - * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. - * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); - -/*! ZSTD_compress_advanced() : - * Note : this function is now DEPRECATED. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); - -/*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams); - - -/*! ZSTD_CCtx_loadDictionary_byReference() : - * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. - * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_loadDictionary_advanced() : - * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_CCtx_refPrefix_advanced() : - * Same as ZSTD_CCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/* === experimental parameters === */ -/* these parameters can be used with ZSTD_setParameter() - * they are not guaranteed to remain supported in the future */ - - /* Enables rsyncable mode, - * which makes compressed files more rsync friendly - * by adding periodic synchronization points to the compressed data. - * The target average block size is ZSTD_c_jobSize / 2. - * It's possible to modify the job size to increase or decrease - * the granularity of the synchronization point. - * Once the jobSize is smaller than the window size, - * it will result in compression ratio degradation. - * NOTE 1: rsyncable mode only works when multithreading is enabled. - * NOTE 2: rsyncable performs poorly in combination with long range mode, - * since it will decrease the effectiveness of synchronization points, - * though mileage may vary. - * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. - * If the selected compression level is already running significantly slower, - * the overall speed won't be significantly impacted. - */ - #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 - -/* Select a compression format. - * The value must be of type ZSTD_format_e. - * See ZSTD_format_e enum definition for details */ -#define ZSTD_c_format ZSTD_c_experimentalParam2 - -/* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ -#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 - -/* Controls whether the contents of a CDict - * are used in place, or copied into the working context. - * Accepts values from the ZSTD_dictAttachPref_e enum. - * See the comments on that enum for an explanation of the feature. */ -#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 - -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. - */ -#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 - -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - -/* User's best guess of source size. - * Hint is not valid when srcSizeHint == 0. - * There is no guarantee that hint is close to actual source size, - * but compression ratio may regress significantly if guess considerably underestimates */ -#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 - -/* Controls whether the new and experimental "dedicated dictionary search - * structure" can be used. This feature is still rough around the edges, be - * prepared for surprising behavior! - * - * How to use it: - * - * When using a CDict, whether to use this feature or not is controlled at - * CDict creation, and it must be set in a CCtxParams set passed into that - * construction (via ZSTD_createCDict_advanced2()). A compression will then - * use the feature or not based on how the CDict was constructed; the value of - * this param, set in the CCtx, will have no effect. - * - * However, when a dictionary buffer is passed into a CCtx, such as via - * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control - * whether the CDict that is created internally can use the feature or not. - * - * What it does: - * - * Normally, the internal data structures of the CDict are analogous to what - * would be stored in a CCtx after compressing the contents of a dictionary. - * To an approximation, a compression using a dictionary can then use those - * data structures to simply continue what is effectively a streaming - * compression where the simulated compression of the dictionary left off. - * Which is to say, the search structures in the CDict are normally the same - * format as in the CCtx. - * - * It is possible to do better, since the CDict is not like a CCtx: the search - * structures are written once during CDict creation, and then are only read - * after that, while the search structures in the CCtx are both read and - * written as the compression goes along. This means we can choose a search - * structure for the dictionary that is read-optimized. - * - * This feature enables the use of that different structure. - * - * Note that some of the members of the ZSTD_compressionParameters struct have - * different semantics and constraints in the dedicated search structure. It is - * highly recommended that you simply set a compression level in the CCtxParams - * you pass into the CDict creation call, and avoid messing with the cParams - * directly. - * - * Effects: - * - * This will only have any effect when the selected ZSTD_strategy - * implementation supports this feature. Currently, that's limited to - * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. - * - * Note that this means that the CDict tables can no longer be copied into the - * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. - * - * In general, you should expect compression to be faster--sometimes very much - * so--and CDict creation to be slightly slower. Eventually, we will probably - * make this mode the default. - */ -#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 - -/* ZSTD_c_stableInBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. - * - * When this flag is enabled zstd won't allocate an input window buffer, - * because the user guarantees it can reference the ZSTD_inBuffer until - * the frame is complete. But, it will still allocate an output buffer - * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also - * avoid the memcpy() from the input buffer to the input window buffer. - * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * - * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find - * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 - -/* ZSTD_c_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells he compressor that the ZSTD_outBuffer will not be resized between - * calls. Specifically: (out.size - out.pos) will never grow. This gives the - * compressor the freedom to say: If the compressed data doesn't fit in the - * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to - * always decompress directly into the output buffer, instead of decompressing - * into an internal buffer and copying to the output buffer. - * - * When this flag is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer. It will still allocate the - * input window buffer (see ZSTD_c_stableInBuffer). - * - * Zstd will check that (out.size - out.pos) never grows and return an error - * if it does. While not strictly necessary, this should prevent surprises. - */ -#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 - -/* ZSTD_c_blockDelimiters - * Default is 0 == ZSTD_sf_noBlockDelimiters. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * - * Designates whether or not the given array of ZSTD_Sequence contains block delimiters - * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. - * See the definition of ZSTD_Sequence for more specifics. - */ -#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 - -/* ZSTD_c_validateSequences - * Default is 0 == disabled. Set to 1 to enable sequence validation. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() - * during function execution. - * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. - * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and - * return an error. - * - */ -#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 - -/*! ZSTD_CCtx_getParameter() : - * Get the requested compression parameter value, selected by enum ZSTD_cParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); - - -/*! ZSTD_CCtx_params : - * Quick howto : - * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). - * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to - * an existing CCtx. - * These parameters will be applied to - * all subsequent frames. - * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. - * - * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() - * for static allocation of CCtx for single-threaded compression. - */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_reset() : - * Reset params to default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_init() : - * Initializes the compression parameters of cctxParams according to - * compression level. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); - -/*! ZSTD_CCtxParams_init_advanced() : - * Initializes the compression and frame parameters of cctxParams according to - * params. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); - -/*! ZSTD_CCtxParams_setParameter() : - * Similar to ZSTD_CCtx_setParameter. - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using - * ZSTD_CCtx_setParametersUsingCCtxParams(). - * @result : a code representing success or failure (which can be tested with - * ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtxParams_getParameter() : - * Similar to ZSTD_CCtx_getParameter. - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); - -/*! ZSTD_CCtx_setParametersUsingCCtxParams() : - * Apply a set of ZSTD_CCtx_params to the compression context. - * This can be done even after compression is started, - * if nbWorkers==0, this will have no impact until a new compression is started. - * if nbWorkers>=1, new parameters will be picked up at next job, - * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); - -/*! ZSTD_compressStream2_simpleArgs() : - * Same as ZSTD_compressStream2(), - * but using only integral types as arguments. - * This variant might be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( - ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos, - ZSTD_EndDirective endOp); - - -/*************************************** -* Advanced decompression functions -***************************************/ - -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * Dictionary content is referenced, and therefore stays in dictBuffer. - * It is important that dictBuffer outlives DDict, - * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_byReference() : - * Same as ZSTD_DCtx_loadDictionary(), - * but references `dict` content instead of copying it into `dctx`. - * This saves memory if `dict` remains around., - * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_advanced() : - * Same as ZSTD_DCtx_loadDictionary(), - * but gives direct control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_refPrefix_advanced() : - * Same as ZSTD_DCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_setMaxWindowSize() : - * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. - * This protects a decoder context from reserving too much memory for itself (potential attack scenario). - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); - -/*! ZSTD_DCtx_getParameter() : - * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); - -/* ZSTD_d_format - * experimental parameter, - * allowing selection between ZSTD_format_e input compression formats - */ -#define ZSTD_d_format ZSTD_d_experimentalParam1 -/* ZSTD_d_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the decompressor, and - * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer - * MUST be large enough to fit the entire decompressed frame. This will be - * checked when the frame content size is known. The data in the ZSTD_outBuffer - * in the range [dst, dst + pos) MUST not be modified during decompression - * or you will get data corruption. - * - * When this flags is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer, but it will still allocate - * an input buffer large enough to fit any compressed block. This will also - * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. - * If you need to avoid the input buffer allocation use the buffer-less - * streaming API. - * - * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, decompression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST - * not be modified during decompression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate - * matches. Normally zstd maintains its own buffer for this purpose, but passing - * this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 - -/* ZSTD_d_forceIgnoreChecksum - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * Tells the decompressor to skip checksum validation during decompression, regardless - * of whether checksumming was specified during compression. This offers some - * slight performance benefits, and may be useful for debugging. - * Param has values of type ZSTD_forceIgnoreChecksum_e - */ -#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 - -/* ZSTD_d_refMultipleDDicts - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * If enabled and dctx is allocated on the heap, then additional memory will be allocated - * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() - * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead - * store all references. At decompression time, the appropriate dictID is selected - * from the set of DDicts based on the dictID in the frame. - * - * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. - * - * Param has values of byte ZSTD_refMultipleDDicts_e - * - * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory - * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. - * Memory is allocated as per ZSTD_DCtx::customMem. - * - * Although this function allocates memory for the table, the user is still responsible for - * memory management of the underlying ZSTD_DDict* themselves. - */ -#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 - - -/*! ZSTD_DCtx_setFormat() : - * Instruct the decoder context about what kind of data to decode next. - * This instruction is mandatory to decode data without a fully-formed header, - * such ZSTD_f_zstd1_magicless for example. - * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); - -/*! ZSTD_decompressStream_simpleArgs() : - * Same as ZSTD_decompressStream(), - * but using only integral types as arguments. - * This can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( - ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos); - - -/******************************************************************** -* Advanced streaming functions -* Warning : most of these functions are now redundant with the Advanced API. -* Once Advanced API reaches "stable" status, -* redundant functions will be deprecated, and then at some point removed. -********************************************************************/ - -/*===== Advanced Streaming compression functions =====*/ - -/*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * pledgedSrcSize must be correct. If it is not known at init time, use - * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, - * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, - int compressionLevel, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * Creates of an internal CDict (incompatible with static CCtx), except if - * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if - * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - int compressionLevel); - -/*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. - * pledgedSrcSize must be correct. - * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); - -/*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize); - -/*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. - * Note that zcs must be init at least once before using ZSTD_resetCStream(). - * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. - * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. - * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, - * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); - - -typedef struct { - unsigned long long ingested; /* nb input bytes read and buffered */ - unsigned long long consumed; /* nb input bytes actually compressed */ - unsigned long long produced; /* nb of compressed bytes generated and buffered */ - unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ - unsigned currentJobID; /* MT only : latest started job nb */ - unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ -} ZSTD_frameProgression; - -/* ZSTD_getFrameProgression() : - * tells how much data has been ingested (read from input) - * consumed (input actually compressed) and produced (output) for current frame. - * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. - * Aggregates progression inside active worker threads. - */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); - -/*! ZSTD_toFlushNow() : - * Tell how many bytes are ready to be flushed immediately. - * Useful for multithreading scenarios (nbWorkers >= 1). - * Probe the oldest active job, defined as oldest job not yet entirely flushed, - * and check its output buffer. - * @return : amount of data stored in oldest job and ready to be flushed immediately. - * if @return == 0, it means either : - * + there is no active job (could be checked with ZSTD_frameProgression()), or - * + oldest job is still actively compressing data, - * but everything it has produced has also been flushed so far, - * therefore flush speed is limited by production speed of oldest job - * irrespective of the speed of concurrent (and newer) jobs. - */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); - - -/*===== Advanced Streaming decompression functions =====*/ - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); - * - * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, ddict); - * - * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); - - -/********************************************************************* -* Buffer-less and synchronous inner streaming functions -* -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. -********************************************************************* */ - -/** - Buffer-less streaming compression (synchronous mode) - - A ZSTD_CCtx object is required to track streaming operations. - Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. - - Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() - - Then, consume your input using ZSTD_compressContinue(). - There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. - - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). - ZSTD_compressContinue() doesn't guarantee recover after a failed compression. - - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). - It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) - - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. - In which case, it will "discard" the relevant memory section from its history. - - Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. - It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. -*/ - -/*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/** - Buffer-less streaming decompression (synchronous mode) - - A ZSTD_DCtx object is required to track streaming operations. - Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. - - First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). - Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. - Data fragment must be large enough to ensure successful decoding. - `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. - errorCode, which can be tested using ZSTD_isError(). - - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, - such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). - Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. - As a consequence, check that values remain within valid application range. - For example, do not allocate memory blindly, check that `windowSize` is within expectation. - Each application can set its own limits, depending on local restrictions. - For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. - - ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. - ZSTD_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference distance. - There are multiple ways to guarantee this condition. - - The most memory efficient way is to use a round buffer of sufficient size. - Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). - In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, - up to the moment there is not enough room left in the buffer to guarantee decoding another full block, - which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. - At which point, decoding can resume from the beginning of the buffer. - Note that already decoded data stored in the buffer should be flushed before being overwritten. - - There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. - - Finally, if you control the compression process, you can also ignore all buffer size rules, - as long as the encoder and decoder progress in "lock-step", - aka use exactly the same buffer sizes, break contiguity at the same place, etc. - - Once buffers are setup, start decompression, with ZSTD_decompressBegin(). - If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). - - Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. - ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. - It can also be an error code, which can be tested with ZSTD_isError(). - - A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). - This information is not required to properly decode a frame. - - == Special case : skippable frames == - - Skippable frames allow integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by decompressor. - The format of skippable frames is as follows : - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. - For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. -*/ - -/*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; - -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - - - - -/* ============================ */ -/** Block level API */ -/* ============================ */ - -/*! - Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). - But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTD_createCCtx() and ZSTD_createDCtx() - - It is necessary to init context before starting - + compression : any ZSTD_compressBegin*() variant, including with dictionary - + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB - + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! - ===> In which case, nothing is produced into `dst` ! - + User __must__ test for such outcome and deal directly with uncompressed data - + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. - Doing so would mess up with statistics history, leading to potential data corruption. - + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! - + In case of multiple successive blocks, should some of them be uncompressed, - decoder must be informed of their existence in order to follow proper history. - Use ZSTD_insertBlock() for such a case. -*/ - -/*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - - -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} -#endif -/**** ended inlining ../zstd.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: huf.h ****/ -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#endif -/**** start inlining xxhash.h ****/ -/* - * xxHash - Extremely Fast Hash algorithm - * Header File - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - - -/* **************************** -* Definitions -******************************/ -/**** skipping file: zstd_deps.h ****/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful if you want to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module anymore. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/* **************************** -* Simple Hash Functions -******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ - - -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ - -/*! State allocation, compatible with dynamic libraries */ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - - -/* hash streaming */ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - - -/* ************************** -* Utils -****************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ -#endif - -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); - - -/* ************************** -* Canonical representation -****************************/ -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); - -#endif /* XXHASH_H_5627135585666179 */ - - - -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ -#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) -#define XXH_STATIC_H_3543687687345 - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ - - -# ifdef XXH_PRIVATE_API -/**** start inlining xxhash.c ****/ -/* - * xxHash - Fast Hash algorithm - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) +/* This file provides custom allocation primitives */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ - defined(__ICCARM__) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for ZSTD_malloc(), ZSTD_free() */ #define ZSTD_DEPS_NEED_MALLOC /**** skipping file: zstd_deps.h ****/ -static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } -static void XXH_free (void* p) { ZSTD_free(p); } -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -#endif -/**** skipping file: xxhash.h ****/ - - -/* ************************************* -* Compiler Specific Options -***************************************/ /**** skipping file: compiler.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** *NOT* inlining ../zstd.h ****/ +#include "zstd.h" /* ZSTD_customMem */ - -/* ************************************* -* Basic Types -***************************************/ -/**** skipping file: mem.h ****/ - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; - -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -#if defined(__ICCARM__) -# include -# define XXH_rotl32(x,r) __ROR(x,(32 - r)) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ************************** -* Utils -****************************/ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - - -/* *************************** -* Simple Hash Functions -*****************************/ - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -/* ************************************************** -* Advanced Hash Functions -****************************************************/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - - -/*** Hash feed ***/ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - - -/* **** XXH64 **** */ - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - if (input != NULL) { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - } - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/* ************************** -* Canonical representation -****************************/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} -/**** ended inlining xxhash.c ****/ -# endif - -#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ - - -#if defined (__cplusplus) -} -#endif -/**** ended inlining xxhash.h ****/ - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ---- static assert (debug) --- */ -#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - - -/*-************************************* -* shared macros -***************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -/** - * Ignore: this is an internal helper. - * - * This is a helper function to help force C99-correctness during compilation. - * Under strict compilation modes, variadic macro arguments can't be empty. - * However, variadic function arguments can be. Using a function therefore lets - * us statically check that at least one (string) argument was passed, - * independent of the compilation flags. - */ -static INLINE_KEYWORD UNUSED_ATTR -void _force_has_format_string(const char *format, ...) { - (void)format; -} - -/** - * Ignore: this is an internal helper. - * - * We want to force this function invocation to be syntactically correct, but - * we don't want to force runtime evaluation of its arguments. - */ -#define _FORCE_HAS_FORMAT_STRING(...) \ - if (0) { \ - _force_has_format_string(__VA_ARGS__); \ - } - -/** - * Return the specified error if the condition evaluates to true. - * - * In debug modes, prints additional information. - * In order to do that (particularly, printing the conditional that failed), - * this can't just wrap RETURN_ERROR(). - */ -#define RETURN_ERROR_IF(cond, err, ...) \ - if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } - -/** - * Unconditionally return the specified error. - * - * In debug modes, prints additional information. - */ -#define RETURN_ERROR(err, ...) \ - do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } while(0); - -/** - * If the provided expression evaluates to an error code, returns that error code. - * - * In debug modes, prints additional information. - */ -#define FORWARD_IF_ERROR(err, ...) \ - do { \ - size_t const err_code = (err); \ - if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return err_code; \ - } \ - } while(0); - - -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1<<12) - -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; - -#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ - -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; - -#define ZSTD_FRAMECHECKSUMSIZE 4 - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 - -#define Litbits 8 -#define MaxLit ((1<= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); - - if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { - /* Handle short offset copies. */ - do { - COPY8(op, ip) - } while (op < oend); - } else { - assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); - /* Separate out the first COPY16() call because the copy length is - * almost certain to be short, so the branches have different - * probabilities. Since it is almost certain to be short, only do - * one COPY16() in the first call. Then, do two calls per loop since - * at that point it is more likely to have a high trip count. - */ -#ifdef __aarch64__ - do { - COPY16(op, ip); - } - while (op < oend); -#else - ZSTD_copy16(op, ip); - if (16 >= length) return; - op += 16; - ip += 16; - do { - COPY16(op, ip); - COPY16(op, ip); - } - while (op < oend); -#endif - } -} - -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - if (length > 0) { - ZSTD_memcpy(dst, src, length); - } - return length; -} - -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 - -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 - -/* Controls whether the input/output buffer is buffered or stable. */ -typedef enum { - ZSTD_bm_buffered = 0, /* Buffer the input/output */ - ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ -} ZSTD_bufferMode_e; - - -/*-******************************************* -* Private declarations -*********************************************/ -typedef struct seqDef_s { - U32 offset; /* Offset code of the sequence */ - U16 litLength; - U16 matchLength; -} seqDef; - -typedef struct { - seqDef* sequencesStart; - seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; - size_t maxNbSeq; - size_t maxNbLit; - - /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength - * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment - * the existing value of the litLength or matchLength by 0x10000. - */ - U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ -} seqStore_t; - -typedef struct { - U32 litLength; - U32 matchLength; -} ZSTD_sequenceLength; - -/** - * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences - * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. - */ -MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) -{ - ZSTD_sequenceLength seqLen; - seqLen.litLength = seq->litLength; - seqLen.matchLength = seq->matchLength + MINMATCH; - if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { - if (seqStore->longLengthID == 1) { - seqLen.litLength += 0xFFFF; - } - if (seqStore->longLengthID == 2) { - seqLen.matchLength += 0xFFFF; - } - } - return seqLen; -} - -/** - * Contains the compressed frame size and an upper-bound for the decompressed frame size. - * Note: before using `compressedSize`, check for errors using ZSTD_isError(). - * similarly, before using `decompressedBound`, check for errors using: - * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` - */ -typedef struct { - size_t compressedSize; - unsigned long long decompressedBound; -} ZSTD_frameSizeInfo; /* decompress & legacy */ - -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H /* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); - -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) { - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); } } - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ - - -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; /* declared here for decompress and fullbench */ - -/*! ZSTD_getcBlockSize() : - * Provides the size of compressed block from block header `src` */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr); - -/*! ZSTD_decodeSeqHeaders() : - * decode sequence header from src */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_CCOMMON_H_MODULE */ -/**** ended inlining zstd_internal.h ****/ +#endif /* ZSTD_ALLOCATIONS_H */ +/**** ended inlining ../common/allocations.h ****/ +/**** skipping file: zstd_deps.h ****/ +/**** skipping file: debug.h ****/ /**** start inlining pool.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -7904,10 +4350,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, #ifndef POOL_H #define POOL_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: zstd_deps.h ****/ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ @@ -7931,10 +4373,16 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, */ void POOL_free(POOL_ctx* ctx); + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + /*! POOL_resize() : * Expands or shrinks pool's number of threads. * This is more efficient than releasing + creating a new context, - * since it tries to preserve and re-use existing threads. + * since it tries to preserve and reuse existing threads. * `numThreads` must be at least 1. * @return : 0 when resize was successful, * !0 (typically 1) if there is an error. @@ -7946,7 +4394,7 @@ int POOL_resize(POOL_ctx* ctx, size_t numThreads); * @return threadpool memory usage * note : compatible with NULL (returns 0 in this case) */ -size_t POOL_sizeof(POOL_ctx* ctx); +size_t POOL_sizeof(const POOL_ctx* ctx); /*! POOL_function : * The function type that can be added to a thread pool. @@ -7963,17 +4411,12 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); /*! POOL_tryAdd() : - * Add the job `function(opaque)` to thread pool _if_ a worker is available. + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. * Returns immediately even if not (does not block). * @return : 1 if successful, 0 if not. */ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); - -#if defined (__cplusplus) -} -#endif - #endif /**** ended inlining pool.h ****/ @@ -8048,7 +4491,7 @@ static void* POOL_thread(void* opaque) { { POOL_job const job = ctx->queue[ctx->queueHead]; ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; ctx->numThreadsBusy++; - ctx->queueEmpty = ctx->queueHead == ctx->queueTail; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); /* Unlock the mutex, signal a pusher, and run the job */ ZSTD_pthread_cond_signal(&ctx->queuePushCond); ZSTD_pthread_mutex_unlock(&ctx->queueMutex); @@ -8058,15 +4501,14 @@ static void* POOL_thread(void* opaque) { /* If the intended queue size was 0, signal after finishing job */ ZSTD_pthread_mutex_lock(&ctx->queueMutex); ctx->numThreadsBusy--; - if (ctx->queueSize == 1) { - ZSTD_pthread_cond_signal(&ctx->queuePushCond); - } + ZSTD_pthread_cond_signal(&ctx->queuePushCond); ZSTD_pthread_mutex_unlock(&ctx->queueMutex); } } /* for (;;) */ assert(0); /* Unreachable */ } +/* ZSTD_createThreadPool() : public access point */ POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { return POOL_create (numThreads, 0); } @@ -8076,7 +4518,8 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { } POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, - ZSTD_customMem customMem) { + ZSTD_customMem customMem) +{ POOL_ctx* ctx; /* Check parameters */ if (!numThreads) { return NULL; } @@ -8088,7 +4531,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, * empty and full queues. */ ctx->queueSize = queueSize + 1; - ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); ctx->queueHead = 0; ctx->queueTail = 0; ctx->numThreadsBusy = 0; @@ -8102,7 +4545,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, } ctx->shutdown = 0; /* Allocate space for the thread handles */ - ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); ctx->threadCapacity = 0; ctx->customMem = customMem; /* Check for errors */ @@ -8135,7 +4578,7 @@ static void POOL_join(POOL_ctx* ctx) { /* Join all of the threads */ { size_t i; for (i = 0; i < ctx->threadCapacity; ++i) { - ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */ + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ } } } @@ -8150,11 +4593,22 @@ void POOL_free(POOL_ctx *ctx) { ZSTD_customFree(ctx, ctx->customMem); } +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { POOL_free (pool); } -size_t POOL_sizeof(POOL_ctx *ctx) { +size_t POOL_sizeof(const POOL_ctx* ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ return sizeof(*ctx) + ctx->queueSize * sizeof(POOL_job) @@ -8171,10 +4625,10 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) return 0; } /* numThreads > threadCapacity */ - { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); if (!threadPool) return 1; /* replace existing thread pool */ - ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t)); ZSTD_customFree(ctx->threads, ctx->customMem); ctx->threads = threadPool; /* Initialize additional threads */ @@ -8219,9 +4673,12 @@ static int isQueueFull(POOL_ctx const* ctx) { } -static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) { - POOL_job const job = {function, opaque}; + POOL_job job; + job.function = function; + job.opaque = opaque; assert(ctx != NULL); if (ctx->shutdown) return; @@ -8275,7 +4732,9 @@ POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); } -POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ (void)numThreads; (void)queueSize; (void)customMem; @@ -8287,6 +4746,11 @@ void POOL_free(POOL_ctx* ctx) { (void)ctx; } +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + int POOL_resize(POOL_ctx* ctx, size_t numThreads) { (void)ctx; (void)numThreads; return 0; @@ -8303,7 +4767,7 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { return 1; } -size_t POOL_sizeof(POOL_ctx* ctx) { +size_t POOL_sizeof(const POOL_ctx* ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ assert(ctx == &g_poolCtx); return sizeof(*ctx); @@ -8313,7 +4777,7 @@ size_t POOL_sizeof(POOL_ctx* ctx) { /**** ended inlining common/pool.c ****/ /**** start inlining common/zstd_common.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -8328,9 +4792,7835 @@ size_t POOL_sizeof(POOL_ctx* ctx) { * Dependencies ***************************************/ #define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ /**** skipping file: error_private.h ****/ -/**** skipping file: zstd_internal.h ****/ +/**** start inlining zstd_internal.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +/**** skipping file: compiler.h ****/ +/**** start inlining cpu.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +/**** skipping file: mem.h ****/ + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16 + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ +/**** ended inlining cpu.h ****/ +/**** skipping file: mem.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: error_private.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** skipping file: ../zstd.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ +/**** skipping file: huf.h ****/ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +/**** start inlining xxhash.h ****/ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Local adaptations for Zstandard */ + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + +#if defined (__cplusplus) +} +#endif + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + +#if defined (__cplusplus) +} /* end of extern "C" */ +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#include /* memcmp, memcpy */ +#include /* ULLONG_MAX */ + +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_STDLIB */ + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#if defined (__cplusplus) +} +#endif + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +#if defined (__cplusplus) +} /* extern "C" */ +#endif +# endif +# endif /* XXH_VSX_BE */ + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + +#if defined (__cplusplus) +extern "C" { +#endif +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386, aarch64 and risc-v. + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + int streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + int dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#endif /* ZSTD_TRACE_H */ +/**** ended inlining zstd_trace.h ****/ +#else +# define ZSTD_TRACE 0 +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#endif /* ZSTD_CCOMMON_H_MODULE */ +/**** ended inlining zstd_internal.h ****/ /*-**************************************** @@ -8361,46 +12651,12 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } /*! ZSTD_getErrorString() : * provides error code string from enum */ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } - - - -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); -} - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; - } - return ZSTD_calloc(1, size); -} - -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) -{ - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } -} /**** ended inlining common/zstd_common.c ****/ /**** start inlining compress/fse_compress.c ****/ /* ****************************************************************** * FSE : Finite State Entropy encoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -8422,7 +12678,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) /* ****************************************************************** * hist : Histogram functions * part of Finite State Entropy project - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -8494,6 +12750,13 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, */ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/*! HIST_add() : + * Lowest level: just add nb of occurrences of characters from @src into @count. + * @count is not reset. @count array is presumed large enough (i.e. 1 KB). + @ This function does not need any additional stack memory. + */ +void HIST_add(unsigned* count, const void* src, size_t srcSize); /**** ended inlining hist.h ****/ /**** skipping file: ../common/bitstream.h ****/ #define FSE_STATIC_LINKING_ONLY @@ -8502,6 +12765,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, #define ZSTD_DEPS_NEED_MALLOC #define ZSTD_DEPS_NEED_MATH64 /**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/bits.h ****/ /* ************************************************************** @@ -8551,13 +12815,14 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; - U32* cumul = (U32*)workSpace; - FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2)); + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ U32 highThreshold = tableSize-1; - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */ + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); /* CTable header */ tableU16[-2] = (U16) tableLog; @@ -8565,7 +12830,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, assert(tableLog < 16); /* required for threshold strategy to work */ /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ #ifdef __clang_analyzer__ ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ @@ -8574,20 +12839,61 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, /* symbol start positions */ { U32 u; cumul[0] = 0; - for (u=1; u <= maxSymbolValue+1; u++) { + for (u=1; u <= maxSV1; u++) { if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ cumul[u] = cumul[u-1] + 1; tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); } else { - cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ } } - cumul[maxSymbolValue+1] = tableSize+1; + cumul[maxSV1] = (U16)(tableSize+1); } /* Spread symbols */ - { U32 position = 0; + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); /* Must have initialized all positions */ + } + } else { + U32 position = 0; U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { + for (symbol=0; symbol highThreshold) position = (position + step) & tableMask; /* Low proba area */ } } - assert(position==0); /* Must have initialized all positions */ } @@ -8620,16 +12925,17 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, case -1: case 1: symbolTT[s].deltaNbBits = (tableLog << 16) - (1< 1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; - symbolTT[s].deltaFindState = total - normalizedCounter[s]; - total += normalizedCounter[s]; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; } } } } #if 0 /* debug : symbol costs */ @@ -8640,32 +12946,26 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, symbol, normalizedCounter[symbol], FSE_getMaxNbBits(symbolTT, symbol), (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); - } - } + } } #endif return 0; } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ - return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol)); -} -#endif - #ifndef FSE_COMMONDEFS_ONLY - /*-************************************************************** * FSE NCount encoding ****************************************************************/ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) { - size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ } @@ -8694,7 +12994,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, /* Init */ remaining = tableSize+1; /* +1 for extra accuracy */ threshold = tableSize; - nbBits = tableLog+1; + nbBits = (int)tableLog+1; while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ if (previousIs0) { @@ -8713,7 +13013,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, } while (symbol >= start+3) { start+=3; - bitStream += 3 << bitCount; + bitStream += 3U << bitCount; bitCount += 2; } bitStream += (symbol-start) << bitCount; @@ -8733,7 +13033,7 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; + bitStream += (U32)count << bitCount; bitCount += nbBits; bitCount -= (count>8); out+= (bitCount+7) /8; - return (out-ostart); + assert(out >= ostart); + return (size_t)(out-ostart); } @@ -8782,21 +13083,11 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, * FSE Compression Code ****************************************************************/ -FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)ZSTD_malloc(size); -} - -void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } - /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; assert(srcSize > 1); /* Not supported, RLE should be used instead */ return minBits; @@ -8804,7 +13095,7 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) { - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; U32 tableLog = maxTableLog; U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); assert(srcSize > 1); /* Not supported, RLE should be used instead */ @@ -8972,40 +13263,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, return tableLog; } - -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) -{ - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; - - /* Build table */ - for (s=0; s not compressible */ - if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - } - - tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) ); - - /* Write table description header */ - { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); - op += nc_err; - } - - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } - - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; - - return op-ostart; -} - -typedef struct { - FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - union { - U32 hist_wksp[HIST_WKSP_SIZE_U32]; - BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; - } workspace; -} fseWkspMax_t; - -size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) -{ - fseWkspMax_t scratchBuffer; - DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); -} - -size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); -} -#endif - #endif /* FSE_COMMONDEFS_ONLY */ /**** ended inlining compress/fse_compress.c ****/ /**** start inlining compress/hist.c ****/ /* ****************************************************************** * hist : Histogram functions * part of Finite State Entropy project - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -9209,6 +13392,16 @@ unsigned HIST_isError(size_t code) { return ERR_isError(code); } /*-************************************************************** * Histogram functions ****************************************************************/ +void HIST_add(unsigned* count, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + + while (ip= 2 + +static size_t showU32(const U32* arr, size_t size) { - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + size_t u; + for (u=0; u= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} + + /* HUF_compressWeights() : * Same as FSE_compress(), but dedicated to huff0's weights compression. * The use case needs much less stack memory. * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. */ #define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t +HUF_compressWeights(void* dst, size_t dstSize, + const void* weightTable, size_t wtSize, + void* workspace, size_t workspaceSize) { BYTE* const ostart = (BYTE*) dst; BYTE* op = ostart; @@ -9433,33 +13713,30 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight unsigned maxSymbolValue = HUF_TABLELOG_MAX; U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); - FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; - - unsigned count[HUF_TABLELOG_MAX+1]; - S16 norm[HUF_TABLELOG_MAX+1]; + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); /* init conditions */ if (wtSize <= 1) return 0; /* Not compressible */ /* Scan input and build symbol stats */ - { unsigned const maxCount = HIST_count_simple(count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ } tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); /* Write table description header */ - { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), norm, maxSymbolValue, tableLog) ); + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); op += hSize; } /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, CTable) ); + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } @@ -9467,30 +13744,94 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight return (size_t)(op-ostart); } - -/*! HUF_writeCTable() : - `CTable` : Huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) +static size_t HUF_getNbBits(HUF_CElt elt) { + return elt & 0xFF; +} + +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} + +static size_t HUF_getValue(HUF_CElt elt) +{ + return elt & ~(size_t)0xFF; +} + +static size_t HUF_getValueFast(HUF_CElt elt) +{ + return elt; +} + +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} + +static void HUF_setValue(HUF_CElt* elt, size_t value) +{ + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); + } +} + +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable) +{ + HUF_CTableHeader header; + ZSTD_memcpy(&header, ctable, sizeof(header)); + return header; +} + +static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue) +{ + HUF_CTableHeader header; + HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header)); + ZSTD_memset(&header, 0, sizeof(header)); + assert(tableLog < 256); + header.tableLog = (BYTE)tableLog; + assert(maxSymbolValue < 256); + header.maxSymbolValue = (BYTE)maxSymbolValue; + ZSTD_memcpy(ctable, &header, sizeof(header)); +} + +typedef struct { + HUF_CompressWeightsWksp wksp; BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + HUF_CElt const* const ct = CTable + 1; BYTE* op = (BYTE*)dst; U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); - /* check conditions */ + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); + + assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue); + assert(HUF_readCTableHeader(CTable).tableLog == huffLog); + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); /* convert to weight */ - bitsToWeight[0] = 0; + wksp->bitsToWeight[0] = 0; for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; /* attempt weights compression by FSE */ - { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) ); + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ op[0] = (BYTE)hSize; return hSize+1; @@ -9500,9 +13841,9 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); return ((maxSymbolValue+1)/2) + 1; } @@ -9513,6 +13854,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 tableLog = 0; U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; /* get symbol weights */ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); @@ -9522,6 +13864,10 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = nbSymbols - 1; + + HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr); + /* Prepare base value per rank */ { U32 n, nextRankStart = 0; for (n=1; n<=tableLog; n++) { @@ -9533,13 +13879,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void /* fill nbBits */ { U32 n; for (n=0; nn=tableLog+1 */ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; - { U32 n; for (n=0; n>= 1; } } /* assign value within rank, symbol order */ - { U32 n; for (n=0; n HUF_readCTableHeader(CTable).maxSymbolValue) + return 0; + return (U32)HUF_getNbBits(ct[symbolValue]); } -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - /** * HUF_setMaxHeight(): - * Enforces maxNbBits on the Huffman tree described in huffNode. + * Try to enforce @targetNbBits on the Huffman tree described in @huffNode. * - * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts - * the tree to so that it is a valid canonical Huffman tree. + * It attempts to convert all nodes with nbBits > @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. * * @pre The sum of the ranks of each symbol == 2^largestBits, * where largestBits == huffNode[lastNonNull].nbBits. * @post The sum of the ranks of each symbol == 2^largestBits, - * where largestBits is the return value <= maxNbBits. + * where largestBits is the return value (expected <= targetNbBits). * - * @param huffNode The Huffman tree modified in place to enforce maxNbBits. + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. * @param lastNonNull The symbol with the lowest count in the Huffman tree. - * @param maxNbBits The maximum allowed number of bits, which the Huffman tree + * @param targetNbBits The allowed number of bits, which the Huffman tree * may not respect. After this function the Huffman tree will - * respect maxNbBits. - * @return The maximum number of bits of the Huffman tree after adjustment, - * necessarily no more than maxNbBits. + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. */ -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) { const U32 largestBits = huffNode[lastNonNull].nbBits; - /* early exit : no elt > maxNbBits, so the tree is already valid. */ - if (largestBits <= maxNbBits) return largestBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; + + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); /* there are several too large elements (at least >= 2) */ { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); + const U32 baseCost = 1 << (largestBits - targetNbBits); int n = (int)lastNonNull; - /* Adjust any ranks > maxNbBits to maxNbBits. + /* Adjust any ranks > targetNbBits to targetNbBits. * Compute totalCost, which is how far the sum of the ranks is * we are over 2^largestBits after adjust the offending ranks. */ - while (huffNode[n].nbBits > maxNbBits) { + while (huffNode[n].nbBits > targetNbBits) { totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; + huffNode[n].nbBits = (BYTE)targetNbBits; n--; } - /* n stops at huffNode[n].nbBits <= maxNbBits */ - assert(huffNode[n].nbBits <= maxNbBits); - /* n end at index of smallest symbol using < maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) --n; + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; - /* renorm totalCost from 2^largestBits to 2^maxNbBits + /* renorm totalCost from 2^largestBits to 2^targetNbBits * note : totalCost is necessarily a multiple of baseCost */ - assert((totalCost & (baseCost - 1)) == 0); - totalCost >>= (largestBits - maxNbBits); + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); assert(totalCost > 0); /* repay normalized cost */ @@ -9628,19 +13971,19 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* Get pos of last (smallest = lowest cum. count) symbol per rank */ ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; + { U32 currentNbBits = targetNbBits; int pos; for (pos=n ; pos >= 0; pos--) { if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = (U32)pos; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; } } while (totalCost > 0) { /* Try to reduce the next power of 2 above totalCost because we * gain back half the rank. */ - U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { U32 const highPos = rankLast[nBitsToDecrease]; U32 const lowPos = rankLast[nBitsToDecrease-1]; @@ -9680,7 +14023,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) rankLast[nBitsToDecrease] = noSymbol; else { rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ } } /* while (totalCost > 0) */ @@ -9692,11 +14035,11 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) * TODO. */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - /* special case : no rank 1 symbol (using maxNbBits-1); - * let's create one from largest rank 0 (using maxNbBits). + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). */ if (rankLast[1] == noSymbol) { - while (huffNode[n].nbBits == maxNbBits) n--; + while (huffNode[n].nbBits == targetNbBits) n--; huffNode[n+1].nbBits--; assert(n >= 0); rankLast[1] = (U32)(n+1); @@ -9710,26 +14053,122 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) } /* repay normalized cost */ } /* there are several too large elements (at least >= 2) */ - return maxNbBits; + return targetNbBits; } typedef struct { - U32 base; - U32 curr; + U16 base; + U16 curr; } rankPos; -typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; -#define RANK_POSITION_TABLE_SIZE 32 +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 typedef struct { huffNodeTable huffNodeTbl; rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; } HUF_buildCTable_wksp_tables; +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. + */ +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) + +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; +} + +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; +} + +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; +} + +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. + */ +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } +} + /** * HUF_sort(): * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. * * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. * Must have (maxSymbolValue + 1) entries. @@ -9737,42 +14176,51 @@ typedef struct { * @param[in] maxSymbolValue Maximum symbol value. * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. */ -static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition) -{ - int n; - int const maxSymbolValue1 = (int)maxSymbolValue + 1; +static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { + U32 n; + U32 const maxSymbolValue1 = maxSymbolValue+1; /* Compute base and set curr to base. - * For symbol s let lowerRank = BIT_highbit32(count[n]+1) and rank = lowerRank + 1. - * Then 2^lowerRank <= count[n]+1 <= 2^rank. + * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. + * See HUF_getIndex to see bucketing strategy. * We attribute each symbol to lowerRank's base value, because we want to know where * each rank begins in the output, so for rank R we want to count ranks R+1 and above. */ ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); for (n = 0; n < maxSymbolValue1; ++n) { - U32 lowerRank = BIT_highbit32(count[n] + 1); + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); rankPosition[lowerRank].base++; } + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { rankPosition[n-1].base += rankPosition[n].base; rankPosition[n-1].curr = rankPosition[n-1].base; } - /* Sort */ + + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ for (n = 0; n < maxSymbolValue1; ++n) { U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rankPosition[r].curr++; - /* Insert into the correct position in the rank. - * We have at most 256 symbols, so this insertion should be fine. - */ - while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) { - huffNode[pos] = huffNode[pos-1]; - pos--; - } + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); huffNode[pos].count = c; huffNode[pos].byte = (BYTE)n; } + + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); } @@ -9796,6 +14244,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) int lowS, lowN; int nodeNb = STARTNODE; int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); /* init for parents */ nonNullRank = (int)maxSymbolValue; while(huffNode[nonNullRank].count == 0) nonNullRank--; @@ -9822,6 +14271,8 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) for (n=0; n<=nonNullRank; n++) huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); + return nonNullRank; } @@ -9837,6 +14288,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) */ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) { + HUF_CElt* const ct = CTable + 1; /* fill result into ctable (val, nbBits) */ int n; U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; @@ -9852,127 +14304,381 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i min >>= 1; } } for (n=0; nhuffNodeTbl; nodeElt* const huffNode = huffNode0+1; int nonNullRank; + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); + /* safety checks */ - if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) - return ERROR(workSpace_tooSmall); + return ERROR(workSpace_tooSmall); if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); + return ERROR(maxSymbolValue_tooLarge); ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); /* sort, decreasing order */ HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); /* build tree */ nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); - /* enforce maxTableLog */ + /* determine and enforce maxTableLog */ maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - HUF_buildCTableFromTree(tree, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); return maxNbBits; } size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CElt const* ct = CTable + 1; size_t nbBits = 0; int s; for (s = 0; s <= (int)maxSymbolValue; ++s) { - nbBits += CTable[s].nbBits * count[s]; + nbBits += HUF_getNbBits(ct[s]) * count[s]; } return nbBits >> 3; } int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { - int bad = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - bad |= (count[s] != 0) & (CTable[s].nbBits == 0); - } - return !bad; + HUF_CTableHeader header = HUF_readCTableHeader(CTable); + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + + assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX); + + if (header.maxSymbolValue < maxSymbolValue) + return 0; + + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; } size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } -FORCE_INLINE_TEMPLATE void -HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) + +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; + + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; + +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) { - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; } -#define HUF_FLUSHBITS(s) BIT_flushBits(s) +/*! HUF_addBits(): + * Adds the symbol stored in HUF_CElt elt to the bitstream. + * + * @param elt The element we're adding. This is a (nbBits, value) pair. + * See the HUF_CStream_t docs for the format. + * @param idx Insert into the bitstream at this idx. + * @param kFast This is a template parameter. If the bitstream is guaranteed + * to have at least 4 unused bits after this call it may be 1, + * otherwise it must be 0. HUF_addBits() is faster when fast is set. + */ +FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) +{ + assert(idx <= 1); + assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); + /* This is efficient on x86-64 with BMI2 because shrx + * only reads the low 6 bits of the register. The compiler + * knows this and elides the mask. When fast is set, + * every operation can use the same value loaded from elt. + */ + bitC->bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} + +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. + */ +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} + +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} + +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. + */ +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} + +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} + +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); + +} + +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) FORCE_INLINE_TEMPLATE size_t HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) { + U32 const tableLog = HUF_readCTableHeader(CTable).tableLog; + HUF_CElt const* ct = CTable + 1; const BYTE* ip = (const BYTE*) src; BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - BIT_CStream_t bitC; + HUF_CStream_t bitC; /* init */ if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const initErr = BIT_initCStream(&bitC, op, (size_t)(oend-op)); + { BYTE* op = ostart; + size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); if (HUF_isError(initErr)) return 0; } - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) - { - case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); - HUF_FLUSHBITS_2(&bitC); - /* fall-through */ - case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); - HUF_FLUSHBITS_1(&bitC); - /* fall-through */ - case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); - HUF_FLUSHBITS(&bitC); - /* fall-through */ - case 0 : /* fall-through */ - default: break; + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } } + assert(bitC.ptr <= bitC.endPtr); - for (; n>0; n-=4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n- 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); + return HUF_closeCStream(&bitC); } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) @@ -9991,9 +14697,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, static size_t HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) + const HUF_CElt* CTable, const int flags) { - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); } return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); @@ -10004,24 +14710,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, static size_t HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) + const HUF_CElt* CTable, const int flags) { - (void)bmi2; + (void)flags; return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); } #endif -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) { - return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); } - static size_t HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, int bmi2) + const HUF_CElt* CTable, int flags) { size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ const BYTE* ip = (const BYTE*) src; @@ -10035,27 +14740,24 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, op += 6; /* jumpTable */ assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart, (U16)cSize); op += cSize; } ip += segmentSize; assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart+2, (U16)cSize); op += cSize; } ip += segmentSize; assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); - if (cSize==0) return 0; - assert(cSize <= 65535); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart+4, (U16)cSize); op += cSize; } @@ -10063,17 +14765,17 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, ip += segmentSize; assert(op <= oend); assert(ip <= iend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); - if (cSize==0) return 0; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; op += cSize; } return (size_t)(op-ostart); } -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) { - return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); } typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; @@ -10081,11 +14783,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; static size_t HUF_compressCTable_internal( BYTE* const ostart, BYTE* op, BYTE* const oend, const void* src, size_t srcSize, - HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) { size_t const cSize = (nbStreams==HUF_singleStream) ? - HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : - HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); if (HUF_isError(cSize)) { return cSize; } if (cSize==0) { return 0; } /* uncompressible */ op += cSize; @@ -10097,32 +14799,113 @@ static size_t HUF_compressCTable_internal( typedef struct { unsigned count[HUF_SYMBOLVALUE_MAX + 1]; - HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1]; - HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; } HUF_compress_tables_t; +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + + { size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + } + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + /* HUF_compress_internal() : * `workSpace_align4` must be aligned on 4-bytes boundaries, - * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */ + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ static size_t HUF_compress_internal (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, HUF_nbStreams_e nbStreams, - void* workSpace_align4, size_t wkspSize, - HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, - const int bmi2) + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) { - HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4; + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstSize; BYTE* op = ostart; - HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE); - assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */ + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); /* checks & inits */ - if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall); + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); if (!srcSize) return 0; /* Uncompressed */ if (!dstSize) return 0; /* cannot fit anything within dst budget */ if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ @@ -10132,17 +14915,34 @@ HUF_compress_internal (void* dst, size_t dstSize, if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; /* Heuristic : If old table is valid, use it for small inputs */ - if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; + } + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; + } + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } /* Scan input and build symbol stats */ - { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) ); + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); /* Check validity of previous table */ if ( repeat @@ -10151,26 +14951,25 @@ HUF_compress_internal (void* dst, size_t dstSize, *repeat = HUF_repeat_none; } /* Heuristic : use existing table for small inputs */ - if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); } /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, maxSymbolValue, huffLog, - &table->buildCTable_wksp, sizeof(table->buildCTable_wksp)); + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); CHECK_F(maxBits); huffLog = (U32)maxBits; - /* Zero unused symbols in CTable, so we can check it for validity */ - ZSTD_memset(table->CTable + (maxSymbolValue + 1), 0, - sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt))); + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); } /* Write table description header */ - { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table->CTable, maxSymbolValue, huffLog) ); + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); /* Check if using previous huffman table is beneficial */ if (repeat && *repeat != HUF_repeat_none) { size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); @@ -10178,7 +14977,7 @@ HUF_compress_internal (void* dst, size_t dstSize, if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); } } /* Use the new huffman table */ @@ -10190,98 +14989,42 @@ HUF_compress_internal (void* dst, size_t dstSize, } return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, table->CTable, bmi2); -} - - -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_singleStream, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/); + nbStreams, table->CTable, flags); } size_t HUF_compress1X_repeat (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) { + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, hufTable, - repeat, preferRepeat, bmi2); + repeat, flags); } /* HUF_compress4X_repeat(): * compress input using 4 streams. - * provide workspace to generate compression tables */ -size_t HUF_compress4X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_fourStreams, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/); -} - -/* HUF_compress4X_repeat(): - * compress input using 4 streams. - * re-use an existing huffman compression table */ + * consider skipping quickly + * reuse an existing huffman compression table */ size_t HUF_compress4X_repeat (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) { + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, - hufTable, repeat, preferRepeat, bmi2); + hufTable, repeat, flags); } - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/** HUF_buildCTable() : - * @return : maxNbBits - * Note : count is used before tree is written, so they can safely overlap - */ -size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) -{ - HUF_buildCTable_wksp_tables workspace; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); -} - -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; - return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; - return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); -} -#endif /**** ended inlining compress/huf_compress.c ****/ /**** start inlining compress/zstd_compress_literals.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10295,7 +15038,7 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi ***************************************/ /**** start inlining zstd_compress_literals.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10309,7 +15052,7 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi /**** start inlining zstd_compress_internal.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10329,163 +15072,9 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi * Dependencies ***************************************/ /**** skipping file: ../common/zstd_internal.h ****/ -/**** start inlining ../common/zstd_trace.h ****/ -/* - * Copyright (c) 2016-2021, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_TRACE_H -#define ZSTD_TRACE_H - -#if defined (__cplusplus) -extern "C" { -#endif - -#include - -/* weak symbol support */ -#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \ - !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ - !defined(__CYGWIN__) -# define ZSTD_HAVE_WEAK_SYMBOLS 1 -#else -# define ZSTD_HAVE_WEAK_SYMBOLS 0 -#endif -#if ZSTD_HAVE_WEAK_SYMBOLS -# define ZSTD_WEAK_ATTR __attribute__((__weak__)) -#else -# define ZSTD_WEAK_ATTR -#endif - -/* Only enable tracing when weak symbols are available. */ -#ifndef ZSTD_TRACE -# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS -#endif - -#if ZSTD_TRACE - -struct ZSTD_CCtx_s; -struct ZSTD_DCtx_s; -struct ZSTD_CCtx_params_s; - -typedef struct { - /** - * ZSTD_VERSION_NUMBER - * - * This is guaranteed to be the first member of ZSTD_trace. - * Otherwise, this struct is not stable between versions. If - * the version number does not match your expectation, you - * should not interpret the rest of the struct. - */ - unsigned version; - /** - * Non-zero if streaming (de)compression is used. - */ - unsigned streaming; - /** - * The dictionary ID. - */ - unsigned dictionaryID; - /** - * Is the dictionary cold? - * Only set on decompression. - */ - unsigned dictionaryIsCold; - /** - * The dictionary size or zero if no dictionary. - */ - size_t dictionarySize; - /** - * The uncompressed size of the data. - */ - size_t uncompressedSize; - /** - * The compressed size of the data. - */ - size_t compressedSize; - /** - * The fully resolved CCtx parameters (NULL on decompression). - */ - struct ZSTD_CCtx_params_s const* params; - /** - * The ZSTD_CCtx pointer (NULL on decompression). - */ - struct ZSTD_CCtx_s const* cctx; - /** - * The ZSTD_DCtx pointer (NULL on compression). - */ - struct ZSTD_DCtx_s const* dctx; -} ZSTD_Trace; - -/** - * A tracing context. It must be 0 when tracing is disabled. - * Otherwise, any non-zero value returned by a tracing begin() - * function is presented to any subsequent calls to end(). - * - * Any non-zero value is treated as tracing is enabled and not - * interpreted by the library. - * - * Two possible uses are: - * * A timestamp for when the begin() function was called. - * * A unique key identifying the (de)compression, like the - * address of the [dc]ctx pointer if you need to track - * more information than just a timestamp. - */ -typedef unsigned long long ZSTD_TraceCtx; - -/** - * Trace the beginning of a compression call. - * @param cctx The dctx pointer for the compression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). - */ -ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx); - -/** - * Trace the end of a compression call. - * @param ctx The return value of ZSTD_trace_compress_begin(). - * @param trace The zstd tracing info. - */ -void ZSTD_trace_compress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); - -/** - * Trace the beginning of a decompression call. - * @param dctx The dctx pointer for the decompression. - * It can be used as a key to map begin() to end(). - * @returns Non-zero if tracing is enabled. The return value is - * passed to ZSTD_trace_compress_end(). - */ -ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx); - -/** - * Trace the end of a decompression call. - * @param ctx The return value of ZSTD_trace_decompress_begin(). - * @param trace The zstd tracing info. - */ -void ZSTD_trace_decompress_end( - ZSTD_TraceCtx ctx, - ZSTD_Trace const* trace); - -#endif /* ZSTD_TRACE */ - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_TRACE_H */ -/**** ended inlining ../common/zstd_trace.h ****/ /**** start inlining zstd_cwksp.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -10500,11 +15089,10 @@ void ZSTD_trace_decompress_end( /*-************************************* * Dependencies ***************************************/ +/**** skipping file: ../common/allocations.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ - -#if defined (__cplusplus) -extern "C" { -#endif +/**** skipping file: ../common/portability_macros.h ****/ +/**** skipping file: ../common/compiler.h ****/ /*-************************************* * Constants @@ -10521,13 +15109,18 @@ extern "C" { #define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 #endif + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + /*-************************************* * Structures ***************************************/ typedef enum { ZSTD_cwksp_alloc_objects, - ZSTD_cwksp_alloc_buffers, - ZSTD_cwksp_alloc_aligned + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers } ZSTD_cwksp_alloc_phase_e; /** @@ -10580,8 +15173,8 @@ typedef enum { * * Workspace Layout: * - * [ ... workspace ... ] - * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] * * The various objects that live in the workspace are divided into the * following categories, and are allocated separately: @@ -10603,10 +15196,20 @@ typedef enum { * - Tables: these are any of several different datastructures (hash tables, * chain tables, binary trees) that all respect a common format: they are * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). - * Their sizes depend on the cparams. + * Their sizes depend on the cparams. These tables are 64-byte aligned. * - * - Aligned: these buffers are used for various purposes that require 4 byte - * alignment, but don't require any initialization before they're used. + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. * * - Buffers: these buffers are used for various purposes that don't require * any alignment or initialization before they're used. This means they can @@ -10618,9 +15221,9 @@ typedef enum { * correctly packed into the workspace buffer. That order is: * * 1. Objects - * 2. Buffers - * 3. Aligned - * 4. Tables + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables * * Attempts to reserve objects of different types out of order will fail. */ @@ -10632,6 +15235,7 @@ typedef struct { void* tableEnd; void* tableValidEnd; void* allocStart; + void* initOnceStart; BYTE allocFailed; int workspaceOversizedDuration; @@ -10644,6 +15248,7 @@ typedef struct { ***************************************/ MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { (void)ws; @@ -10653,14 +15258,29 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { assert(ws->tableEnd <= ws->allocStart); assert(ws->tableValidEnd <= ws->allocStart); assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); + (void)offset; +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif } /** * Align must be a power of 2. */ -MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) { size_t const mask = align - 1; - assert((align & mask) == 0); + assert(ZSTD_isPower2(align)); return (size + mask) & ~mask; } @@ -10673,6 +15293,8 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { * Since tables aren't currently redzoned, you don't need to call through this * to figure out how much space you need for the matchState tables. Everything * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size(). */ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { if (size == 0) @@ -10684,58 +15306,68 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { #endif } -MEM_STATIC void ZSTD_cwksp_internal_advance_phase( - ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - ws->tableValidEnd = ws->objectEnd; - } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); - if (ws->allocStart < ws->tableValidEnd) { - ws->tableValidEnd = ws->allocStart; - } - } - ws->phase = phase; - } +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment)); } /** - * Returns whether this object/buffer/etc was allocated in this workspace. + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". */ -MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { - return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) { + return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES); +} + +/** + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). + */ +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert(ZSTD_isPower2(alignBytes)); + assert(bytes < alignBytes); + return bytes; +} + +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) +{ + char* endPtr = (char*)ws->workspaceEnd; + assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES)); + endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES); + return (void*)endPtr; } /** * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. */ -MEM_STATIC void* ZSTD_cwksp_reserve_internal( - ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - void* alloc; - void* bottom = ws->tableEnd; - ZSTD_cwksp_internal_advance_phase(ws, phase); - alloc = (BYTE *)ws->allocStart - bytes; - - if (bytes == 0) - return NULL; - -#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* over-reserve space */ - alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#endif - - DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving [0x%p]:%zd bytes; %zd bytes remaining", alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); ZSTD_cwksp_assert_internal_consistency(ws); assert(alloc >= bottom); @@ -10744,17 +15376,87 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->allocFailed = 1; return NULL; } + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ if (alloc < ws->tableValidEnd) { ws->tableValidEnd = alloc; } ws->allocStart = alloc; + return alloc; +} + +/** + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error + */ +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) +{ + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void *const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void *const objectEnd = (BYTE *) alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + } + } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); + } + return 0; +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) +{ + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; + } + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); #if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on * either size. */ - alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; - if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - __asan_unpoison_memory_region(alloc, bytes); + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); + } } #endif @@ -10764,33 +15466,79 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( /** * Reserves and returns unaligned memory. */ -MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) +{ return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); } /** - * Reserves and returns memory sized on and aligned on sizeof(unsigned). + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). */ -MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; + } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; } /** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes) +{ + void* const ptr = ZSTD_cwksp_reserve_internal(ws, + ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + return ptr; +} + +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to reuse them without * memset()-ing them. */ -MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; + void* alloc; + void* end; + void* top; + + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } + } + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); - ZSTD_cwksp_internal_advance_phase(ws, phase); ZSTD_cwksp_assert_internal_consistency(ws); assert(end <= top); if (end > top) { @@ -10806,14 +15554,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { } #endif + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); return alloc; } /** * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization */ -MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* alloc = ws->objectEnd; void* end = (BYTE*)alloc + roundedBytes; @@ -10822,15 +15574,15 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; #endif - DEBUGLOG(5, + DEBUGLOG(4, "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)alloc & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); ZSTD_cwksp_assert_internal_consistency(ws); /* we must be in the first phase, no advance is possible */ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(4, "cwksp: object alloc failed!"); + DEBUGLOG(3, "cwksp: object alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -10841,7 +15593,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { #if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on * either size. */ - alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { __asan_unpoison_memory_region(alloc, bytes); } @@ -10849,18 +15601,41 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { return alloc; } +/** + * with alignment control + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment) +{ + size_t const mask = alignment - 1; + size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0; + void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus); + if (start == NULL) return NULL; + if (surplus == 0) return start; + assert(ZSTD_isPower2(alignment)); + return (void*)(((size_t)start + surplus) & ~mask); +} -MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) +{ DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the table re-use logic is sound, and that we don't + /* To validate that the table reuse logic is sound, and that we don't * access table space that we haven't cleaned, we re-"poison" the table - * space every time we mark it dirty. */ + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ { size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; assert(__msan_test_shadow(ws->objectEnd, size) == -1); - __msan_poison(ws->objectEnd, size); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } } #endif @@ -10888,7 +15663,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { assert(ws->tableValidEnd >= ws->objectEnd); assert(ws->tableValidEnd <= ws->allocStart); if (ws->tableValidEnd < ws->tableEnd) { - ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); } ZSTD_cwksp_mark_tables_clean(ws); } @@ -10897,7 +15672,8 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { * Invalidates table allocations. * All other allocations remain valid. */ -MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) +{ DEBUGLOG(4, "cwksp: clearing tables!"); #if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) @@ -10923,13 +15699,16 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing!"); #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the context re-use logic is sound, and that we don't + /* To validate that the context reuse logic is sound, and that we don't * access stuff that this compression hasn't initialized, we re-"poison" - * the workspace (or at least the non-static, non-table parts of it) - * every time we start a new compression. */ + * the workspace except for the areas in which we expect memory reuse + * without initialization (objects, valid tables area and init once + * memory). */ { - size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; - __msan_poison(ws->tableValidEnd, size); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } } #endif @@ -10945,14 +15724,23 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { #endif ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; } ZSTD_cwksp_assert_internal_consistency(ws); } +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + /** * The provided workspace takes ownership of the buffer [start, start+size). * Any existing values in the workspace are ignored (the previously managed @@ -10965,6 +15753,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); ws->phase = ZSTD_cwksp_alloc_objects; ws->isStatic = isStatic; ZSTD_cwksp_clear(ws); @@ -10983,6 +15772,11 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { void *ptr = ws->workspace; DEBUGLOG(4, "cwksp: freeing workspace"); +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) + if (ptr != NULL && customMem.customFree != NULL) { + __msan_unpoison(ptr, ZSTD_cwksp_sizeof(ws)); + } +#endif ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); ZSTD_customFree(ptr, customMem); } @@ -10996,15 +15790,6 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); } -MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); -} - -MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) - + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); -} - MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } @@ -11013,6 +15798,18 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { * Functions Checking Free Space ***************************************/ +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; +} + + MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); } @@ -11040,16 +15837,12 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( } } -#if defined (__cplusplus) -} -#endif - #endif /* ZSTD_CWKSP_H */ /**** ended inlining zstd_cwksp.h ****/ #ifdef ZSTD_MULTITHREAD /**** start inlining zstdmt_compress.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11061,10 +15854,10 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( #ifndef ZSTDMT_COMPRESS_H #define ZSTDMT_COMPRESS_H - #if defined (__cplusplus) - extern "C" { - #endif - +/* === Dependencies === */ +/**** skipping file: ../common/zstd_deps.h ****/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +/**** skipping file: ../zstd.h ****/ /* Note : This is an internal API. * These APIs used to be exposed with ZSTDLIB_API, @@ -11075,18 +15868,12 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( * otherwise ZSTDMT_createCCtx*() will fail. */ -/* === Dependencies === */ -/**** skipping file: ../common/zstd_deps.h ****/ -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ -/**** skipping file: ../zstd.h ****/ - - /* === Constants === */ -#ifndef ZSTDMT_NBWORKERS_MAX -# define ZSTDMT_NBWORKERS_MAX 200 +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) #endif -#ifndef ZSTDMT_JOBSIZE_MIN -# define ZSTDMT_JOBSIZE_MIN (1 MB) +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) #endif #define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) @@ -11115,8 +15902,11 @@ size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); * Private use only. Init streaming operation. * expects params to be valid. * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. * @return : 0, or an error code */ -size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); @@ -11152,18 +15942,45 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p */ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); - -#if defined (__cplusplus) -} -#endif - #endif /* ZSTDMT_COMPRESS_H */ /**** ended inlining zstdmt_compress.h ****/ #endif +/**** skipping file: ../common/bits.h ****/ +/**** start inlining zstd_preSplit.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ -#if defined (__cplusplus) -extern "C" { -#endif +#ifndef ZSTD_PRESPLIT_H +#define ZSTD_PRESPLIT_H + +#include /* size_t */ + +#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208 + +/* ZSTD_splitBlock(): + * @level must be a value between 0 and 4. + * higher levels spend more energy to detect block boundaries. + * @workspace must be aligned for size_t. + * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE + * note: + * For the time being, this function only accepts full 128 KB blocks. + * Therefore, @blockSize must be == 128 KB. + * While this could be extended to smaller sizes in the future, + * it is not yet clear if this would be useful. TBD. + */ +size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize); + +#endif /* ZSTD_PRESPLIT_H */ +/**** ended inlining zstd_preSplit.h ****/ /*-************************************* * Constants @@ -11175,7 +15992,7 @@ extern "C" { It's not a big deal though : candidate will just be sorted again. Additionally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table reuse with a different strategy. This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ @@ -11200,7 +16017,7 @@ typedef struct { } ZSTD_localDict; typedef struct { - HUF_CElt CTable[HUF_CTABLE_SIZE_U32(255)]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; HUF_repeat repeatMode; } ZSTD_hufCTables_t; @@ -11218,8 +16035,120 @@ typedef struct { ZSTD_fseCTables_t fse; } ZSTD_entropyCTables_t; +/*********************************************** +* Sequences * +***********************************************/ +typedef struct SeqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} SeqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See SeqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + typedef struct { - U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */ + SeqDef* sequencesStart; + SeqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} SeqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_SequenceLength; + +/** + * Returns the ZSTD_SequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_SequenceLength ZSTD_getSequenceLength(SeqStore_t const* seqStore, SeqDef const* seq) +{ + ZSTD_SequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ +typedef struct { + SymbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + SymbolEncodingType_e llType; + SymbolEncodingType_e ofType; + SymbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ U32 len; /* Raw length of match */ } ZSTD_match_t; @@ -11236,28 +16165,29 @@ typedef struct { stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ size_t size; /* The number of sequences. <= capacity. */ size_t capacity; /* The capacity starting from `seq` pointer */ -} rawSeqStore_t; +} RawSeqStore_t; -UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; +UNUSED_ATTR static const RawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; typedef struct { - int price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTD_REP_NUM]; + int price; /* price from beginning of segment to this position */ + U32 off; /* offset of previous match */ + U32 mlen; /* length of previous match */ + U32 litlen; /* nb of literals since previous match */ + U32 rep[ZSTD_REP_NUM]; /* offset history after previous match */ } ZSTD_optimal_t; typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; +#define ZSTD_OPT_SIZE (ZSTD_OPT_NUM+3) typedef struct { /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ unsigned* litFreq; /* table of literals statistics, of size 256 */ unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ - ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ - ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_SIZE */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_SIZE */ U32 litSum; /* nb of literals */ U32 litLengthSum; /* nb of litLength codes */ @@ -11269,7 +16199,7 @@ typedef struct { U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_ParamSwitch_e literalCompressionMode; } optState_t; typedef struct { @@ -11278,15 +16208,24 @@ typedef struct { } ZSTD_compressedBlockState_t; typedef struct { - BYTE const* nextSrc; /* next block here to continue on current prefix */ - BYTE const* base; /* All regular indexes relative to this position */ - BYTE const* dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more valid data */ + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ } ZSTD_window_t; -typedef struct ZSTD_matchState_t ZSTD_matchState_t; -struct ZSTD_matchState_t { +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_MatchState_t ZSTD_MatchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_MatchState_t { ZSTD_window_t window; /* State for window round buffer management */ U32 loadedDictEnd; /* index of end of dictionary, within context's referential. * When loadedDictEnd != 0, a dictionary is in use, and still valid. @@ -11297,22 +16236,44 @@ struct ZSTD_matchState_t { */ U32 nextToUpdate; /* index from which to continue table update */ U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for reuse of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ + U32* hashTable; U32* hashTable3; U32* chainTable; + + int forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + int dedicatedDictSearch; /* Indicates whether this matchState is using the * dedicated dictionary search structure. */ optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t* dictMatchState; + const ZSTD_MatchState_t* dictMatchState; ZSTD_compressionParameters cParams; - const rawSeqStore_t* ldmSeqStore; + const RawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; }; typedef struct { ZSTD_compressedBlockState_t* prevCBlock; ZSTD_compressedBlockState_t* nextCBlock; - ZSTD_matchState_t matchState; + ZSTD_MatchState_t matchState; } ZSTD_blockState_t; typedef struct { @@ -11339,7 +16300,7 @@ typedef struct { } ldmState_t; typedef struct { - U32 enableLdm; /* 1 if enable long distance matching */ + ZSTD_ParamSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ U32 hashLog; /* Log size of hashTable */ U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ U32 minMatchLength; /* Minimum match length */ @@ -11370,7 +16331,7 @@ struct ZSTD_CCtx_params_s { * There is no guarantee that hint is close to actual source size */ ZSTD_dictAttachPref_e attachDictPref; - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_ParamSwitch_e literalCompressionMode; /* Multithreading: used to pass parameters to mtctx */ int nbWorkers; @@ -11389,15 +16350,54 @@ struct ZSTD_CCtx_params_s { ZSTD_bufferMode_e outBufferMode; /* Sequence compression API */ - ZSTD_sequenceFormat_e blockDelimiters; + ZSTD_SequenceFormat_e blockDelimiters; int validateSequences; + /* Block splitting + * @postBlockSplitter executes split analysis after sequences are produced, + * it's more accurate but consumes more resources. + * @preBlockSplitter_level splits before knowing sequences, + * it's more approximative but also cheaper. + * Valid @preBlockSplitter_level values range from 0 to 6 (included). + * 0 means auto, 1 means do not split, + * then levels are sorted in increasing cpu budget, from 2 (fastest) to 6 (slowest). + * Highest @preBlockSplitter_level combines well with @postBlockSplitter. + */ + ZSTD_ParamSwitch_e postBlockSplitter; + int preBlockSplitter_level; + + /* Adjust the max block size*/ + size_t maxBlockSize; + + /* Param for deciding whether to use row-based matchfinder */ + ZSTD_ParamSwitch_e useRowMatchFinder; + + /* Always load a dictionary in ext-dict mode (not prefix mode)? */ + int deterministicRefPrefix; + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; + + /* Controls prefetching in some dictMatchState matchfinders */ + ZSTD_ParamSwitch_e prefetchCDictTables; + + /* Controls whether zstd will fall back to an internal matchfinder + * if the external matchfinder returns an error code. */ + int enableMatchFinderFallback; + + /* Parameters for the external sequence producer API. + * Users set these parameters through ZSTD_registerSequenceProducer(). + * It is not possible to set these parameters individually through the public API. */ + void* extSeqProdState; + ZSTD_sequenceProducer_F extSeqProdFunc; + + /* Controls repcode search in external sequence parsing */ + ZSTD_ParamSwitch_e searchForExternalRepcodes; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) #define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE) +#define TMP_WORKSPACE_SIZE (MAX(ENTROPY_WORKSPACE_SIZE, ZSTD_SLIPBLOCK_WORKSPACESIZE)) /** * Indicates whether this compression proceeds directly from user-provided @@ -11409,17 +16409,34 @@ typedef enum { ZSTDb_buffered } ZSTD_buffered_policy_e; +/** + * Struct that contains all elements of block splitter that should be allocated + * in a wksp. + */ +#define ZSTD_MAX_NB_BLOCK_SPLITS 196 +typedef struct { + SeqStore_t fullSeqStoreChunk; + SeqStore_t firstHalfSeqStore; + SeqStore_t secondHalfSeqStore; + SeqStore_t currSeqStore; + SeqStore_t nextSeqStore; + + U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS]; + ZSTD_entropyCTablesMetadata_t entropyMetadata; +} ZSTD_blockSplitCtx; + struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; + ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */ U32 dictID; size_t dictContentSize; ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ - size_t blockSize; + size_t blockSizeMax; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; unsigned long long producedCSize; @@ -11431,15 +16448,16 @@ struct ZSTD_CCtx_s { int isFirstBlock; int initialized; - seqStore_t seqStore; /* sequences storage ptrs */ + SeqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ rawSeq* ldmSequences; /* Storage for the ldm output sequences */ size_t maxNbLdmSequences; - rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ + RawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ ZSTD_blockState_t blockState; - U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ + void* tmpWorkspace; /* used as substitute of stack space - must be aligned for S64 type */ + size_t tmpWkspSize; - /* Wether we are streaming or not */ + /* Whether we are streaming or not */ ZSTD_buffered_policy_e bufferedPolicy; /* streaming */ @@ -11457,6 +16475,7 @@ struct ZSTD_CCtx_s { /* Stable in/out buffer verification */ ZSTD_inBuffer expectedInBuffer; + size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */ size_t expectedOutBufferSize; /* Dictionary */ @@ -11473,9 +16492,17 @@ struct ZSTD_CCtx_s { #if ZSTD_TRACE ZSTD_TraceCtx traceCtx; #endif + + /* Workspace for block splitter */ + ZSTD_blockSplitCtx blockSplitCtx; + + /* Buffer for output from external sequence producer */ + ZSTD_Sequence* extSeqBuf; + size_t extSeqBufCapacity; }; typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; +typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e; typedef enum { ZSTD_noDict = 0, @@ -11497,17 +16524,17 @@ typedef enum { * In this mode we take both the source size and the dictionary size * into account when selecting and adjusting the parameters. */ - ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. + ZSTD_cpm_unknown = 3 /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. * We don't know what these parameters are for. We default to the legacy * behavior of taking both the source size and the dict size into account * when selecting and adjusting parameters. */ -} ZSTD_cParamMode_e; +} ZSTD_CParamMode_e; -typedef size_t (*ZSTD_blockCompressor) ( - ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +typedef size_t (*ZSTD_BlockCompressor_f) ( + ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode); +ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode); MEM_STATIC U32 ZSTD_LLcode(U32 litLength) @@ -11541,31 +16568,6 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; } -typedef struct repcodes_s { - U32 rep[3]; -} repcodes_t; - -MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) -{ - repcodes_t newReps; - if (offset >= ZSTD_REP_NUM) { /* full offset */ - newReps.rep[2] = rep[1]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = offset - ZSTD_REP_MOVE; - } else { /* repcode */ - U32 const repCode = offset + ll0; - if (repCode > 0) { /* note : if repCode==0, no change */ - U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = currentOffset; - } else { /* repCode == 0 */ - ZSTD_memcpy(&newReps, rep, sizeof(newReps)); - } - } - return newReps; -} - /* ZSTD_cParam_withinBounds: * @return 1 if value is within cParam bounds, * 0 otherwise */ @@ -11578,12 +16580,33 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) return 1; } +/* ZSTD_selectAddr: + * @return index >= lowLimit ? candidate : backup, + * tries to force branchless codegen. */ +MEM_STATIC const BYTE* +ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* backup) +{ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__ ( + "cmp %1, %2\n" + "cmova %3, %0\n" + : "+r"(candidate) + : "r"(index), "r"(lowLimit), "r"(backup) + ); + return candidate; +#else + return index >= lowLimit ? candidate : backup; +#endif +} + /* ZSTD_noCompressBlock() : * Writes uncompressed block to dst buffer from given src. * Returns the size of the block */ -MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) { U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, dstSize_tooSmall, "dst buf too small for uncompressed block"); MEM_writeLE24(dst, cBlockHeader24); @@ -11591,7 +16614,8 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi return ZSTD_blockHeaderSize + srcSize; } -MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) { BYTE* const op = (BYTE*)dst; U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); @@ -11610,21 +16634,21 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) { U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); return (srcSize >> minlog) + 2; } -MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) { switch (cctxParams->literalCompressionMode) { - case ZSTD_lcm_huffman: + case ZSTD_ps_enable: return 0; - case ZSTD_lcm_uncompressed: + case ZSTD_ps_disable: return 1; default: assert(0 /* impossible: pre-validated */); - /* fall-through */ - case ZSTD_lcm_auto: + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); } } @@ -11634,7 +16658,9 @@ MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParam * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single * large copies. */ -static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ assert(iend > ilimit_w); if (ip <= ilimit_w) { ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); @@ -11644,14 +16670,69 @@ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const ie while (ip < iend) *op++ = *ip++; } -/*! ZSTD_storeSeq() : - * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. - * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). - * `mlBase` : matchLength - MINMATCH - * Allowed to overread literals up to litLimit. + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ + +/*! ZSTD_storeSeqOnly() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t. + * Literals themselves are not copied, but @litPtr is updated. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH */ -HINT_INLINE UNUSED_ATTR -void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr, + size_t litLength, + U32 offBase, + size_t matchLength) +{ + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + + /* literal Length */ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (UNLIKELY(litLength>0xFFFF)) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = offBase; + + /* match Length */ + assert(matchLength <= ZSTD_BLOCKSIZE_MAX); + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (UNLIKELY(mlBase>0xFFFF)) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH + * Allowed to over-read literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(SeqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase, + size_t matchLength) { BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; BYTE const* const litEnd = literals + litLength; @@ -11659,8 +16740,8 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ { U32 const pos = (U32)((const BYTE*)literals - g_start); - DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); } #endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); @@ -11670,9 +16751,9 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera assert(literals + litLength <= litLimit); if (litEnd <= litLimit_w) { /* Common case we can use wildcopy. - * First copy 16 bytes, because literals are likely short. - */ - assert(WILDCOPY_OVERLENGTH >= 16); + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); ZSTD_copy16(seqStorePtr->lit, literals); if (litLength > 16) { ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); @@ -11682,105 +16763,50 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera } seqStorePtr->lit += litLength; - /* literal Length */ - if (litLength>0xFFFF) { - assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ - seqStorePtr->longLengthID = 1; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength); +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = OFFBASE_TO_OFFSET(offBase); + } else { /* repcode */ + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } } - seqStorePtr->sequences[0].litLength = (U16)litLength; +} - /* match offset */ - seqStorePtr->sequences[0].offset = offCode + 1; +typedef struct repcodes_s { + U32 rep[3]; +} Repcodes_t; - /* match Length */ - if (mlBase>0xFFFF) { - assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ - seqStorePtr->longLengthID = 2; - seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - } - seqStorePtr->sequences[0].matchLength = (U16)mlBase; - - seqStorePtr->sequences++; +MEM_STATIC Repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + Repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase, ll0); + return newReps; } /*-************************************* * Match length counter ***************************************/ -static unsigned ZSTD_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val) >> 3; -# else - unsigned long r = 0; - return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, - 0, 3, 1, 3, 1, 4, 2, 7, - 0, 2, 3, 6, 1, 5, 3, 5, - 1, 3, 4, 4, 2, 5, 6, 7, - 7, 0, 1, 2, 3, 3, 4, 6, - 2, 6, 5, 5, 3, 4, 5, 6, - 7, 1, 2, 4, 6, 4, 4, 5, - 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r=0; - return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, - 3, 2, 2, 1, 3, 2, 0, 1, - 3, 3, 1, 2, 2, 2, 2, 0, - 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _lzcnt_u64(val) >> 3; -# else - unsigned long r = 0; - return _BitScanReverse64(&r, (U64)val) ? (unsigned)(r >> 3) : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r = 0; - return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } } -} - - MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) { const BYTE* const pStart = pIn; @@ -11814,8 +16840,8 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, size_t const matchLength = ZSTD_count(ip, match, vEnd); if (match + matchLength != mEnd) return matchLength; DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength); - DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match); - DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip); + DEBUGLOG(7, "distance from match beginning to end dictionary = %i", (int)(mEnd - match)); + DEBUGLOG(7, "distance from current pos to end buffer = %i", (int)(iEnd - ip)); DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart); DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd)); return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); @@ -11826,32 +16852,43 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, * Hashes ***************************************/ static const U32 prime3bytes = 506832829U; -static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } -MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ +static U32 ZSTD_hash3(U32 u, U32 h, U32 s) { assert(h <= 32); return (((u << (32-24)) * prime3bytes) ^ s) >> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } static const U32 prime4bytes = 2654435761U; -static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } -static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } static const U64 prime5bytes = 889523592379ULL; -static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } -static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } -static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } static const U64 prime7bytes = 58295818150454627ULL; -static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } -static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } -static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + MEM_STATIC FORCE_INLINE_ATTR size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + switch(mls) { default: @@ -11863,6 +16900,24 @@ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) } } +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + /** ZSTD_ipow() : * Return base^exponent. */ @@ -11924,11 +16979,12 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 /*-************************************* * Round buffer management ***************************************/ -#if (ZSTD_WINDOWLOG_MAX_64 > 31) -# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" -#endif -/* Max current allowed */ -#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Max @current value allowed: + * In 32-bit mode: we want to avoid crossing the 2 GB limit, + * reducing risks of side effects in case of signed operations on indexes. + * In 64-bit mode: we want to ensure that adding the maximum job size (512 MB) + * doesn't overflow U32 index capacity (4 GB) */ +#define ZSTD_CURRENT_MAX (MEM_64bits() ? 3500U MB : 2000U MB) /* Maximum chunk size before overflow correction needs to be called again */ #define ZSTD_CHUNKSIZE_MAX \ ( ((U32)-1) /* Maximum ending current index */ \ @@ -11947,6 +17003,13 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) window->dictLimit = end; } +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + /** * ZSTD_window_hasExtDict(): * Returns non-zero if the window has a non-empty extDict. @@ -11961,7 +17024,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) * Inspects the provided matchState and figures out what dictMode should be * passed to the compressor. */ -MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms) { return ZSTD_window_hasExtDict(ms->window) ? ZSTD_extDict : @@ -11970,15 +17033,71 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) ZSTD_noDict; } +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + /** * ZSTD_window_needOverflowCorrection(): * Returns non-zero if the indices are getting too large and need overflow * protection. */ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, void const* srcEnd) { U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } return curr > ZSTD_CURRENT_MAX; } @@ -11990,9 +17109,10 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, * * The least significant cycleLog bits of the indices must remain the same, * which may be 0. Every index up to maxDist in the past must be valid. - * NOTE: (maxDist & cycleMask) must be zero. */ -MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, U32 maxDist, void const* src) { /* preemptive overflow correction: @@ -12014,32 +17134,52 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); - U32 const currentCycle0 = curr & cycleMask; - /* Exclude zero so that newCurrent - maxDist >= 1. */ - U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0; - U32 const newCurrent = currentCycle1 + maxDist; + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); U32 const correction = curr - newCurrent; - assert((maxDist & cycleMask) == 0); + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); assert(curr > newCurrent); - /* Loose bound, should be around 1<<29 (see above) */ - assert(correction > 1<<28); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } window->base += correction; window->dictBase += correction; - if (window->lowLimit <= correction) window->lowLimit = 1; - else window->lowLimit -= correction; - if (window->dictLimit <= correction) window->dictLimit = 1; - else window->dictLimit -= correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } /* Ensure we can still reference the full window. */ assert(newCurrent >= maxDist); - assert(newCurrent - maxDist >= 1); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); /* Ensure that lowLimit and dictLimit didn't underflow. */ assert(window->lowLimit <= newCurrent); assert(window->dictLimit <= newCurrent); + ++window->nbOverflowCorrections; + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, window->lowLimit); return correction; @@ -12073,7 +17213,7 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, const void* blockEnd, U32 maxDist, U32* loadedDictEndPtr, - const ZSTD_matchState_t** dictMatchStatePtr) + const ZSTD_MatchState_t** dictMatchStatePtr) { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; @@ -12118,7 +17258,7 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, const void* blockEnd, U32 maxDist, U32* loadedDictEndPtr, - const ZSTD_matchState_t** dictMatchStatePtr) + const ZSTD_MatchState_t** dictMatchStatePtr) { assert(loadedDictEndPtr != NULL); assert(dictMatchStatePtr != NULL); @@ -12128,10 +17268,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); assert(blockEndIdx >= loadedDictEnd); - if (blockEndIdx > loadedDictEnd + maxDist) { + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { /* On reaching window size, dictionaries are invalidated. * For simplification, if window size is reached anywhere within next block, * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. */ DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); *loadedDictEndPtr = 0; @@ -12144,11 +17289,13 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { ZSTD_memset(window, 0, sizeof(*window)); - window->base = (BYTE const*)""; - window->dictBase = (BYTE const*)""; - window->dictLimit = 1; /* start from 1, so that 1st position is valid */ - window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - window->nextSrc = window->base + 1; /* see issue #1241 */ + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; } /** @@ -12158,8 +17305,11 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { * forget about the extDict. Handles overlap of the prefix and extDict. * Returns non-zero if the segment is contiguous. */ -MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, - void const* src, size_t srcSize) +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_update(ZSTD_window_t* window, + const void* src, size_t srcSize, + int forceNonContiguous) { BYTE const* const ip = (BYTE const*)src; U32 contiguous = 1; @@ -12169,7 +17319,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, assert(window->base != NULL); assert(window->dictBase != NULL); /* Check if blocks follow each other */ - if (src != window->nextSrc) { + if (src != window->nextSrc || forceNonContiguous) { /* not contiguous */ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); @@ -12186,8 +17336,9 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ if ( (ip+srcSize > window->dictBase + window->lowLimit) & (ip < window->dictBase + window->dictLimit)) { - ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; - U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + size_t const highInputIdx = (size_t)((ip + srcSize) - window->dictBase); + U32 const lowLimitMax = (highInputIdx > (size_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + assert(highInputIdx < UINT_MAX); window->lowLimit = lowLimitMax; DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); } @@ -12197,24 +17348,24 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, /** * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. */ -MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog) { - U32 const maxDistance = 1U << windowLog; - U32 const lowestValid = ms->window.lowLimit; - U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; - U32 const isDictionary = (ms->loadedDictEnd != 0); + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't * valid for the entire block. So this check is sufficient to find the lowest valid match index. */ - U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; return matchLowest; } /** * Returns the lowest allowed match index in the prefix. */ -MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; U32 const lowestValid = ms->window.dictLimit; @@ -12227,6 +17378,13 @@ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, return matchLowest; } +/* index_safety_check: + * intentional underflow : ensure repIndex isn't overlapping dict + prefix + * @return 1 if values are not overlapping, + * 0 otherwise */ +MEM_STATIC int ZSTD_index_overlap_check(const U32 prefixLowestIndex, const U32 repIndex) { + return ((U32)((prefixLowestIndex-1) - repIndex) >= 3); +} /* debug functions */ @@ -12260,10 +17418,42 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) #endif +/* Short Cache */ -#if defined (__cplusplus) +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; } -#endif /* =============================================================== * Shared internal declarations @@ -12280,6 +17470,25 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_SequencePosition; + +/* for benchmark */ +size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, + const ZSTD_Sequence* const inSeqs, size_t nbSequences, + int const repcodeResolution); + +typedef struct { + size_t nbSequences; + size_t blockSize; + size_t litSize; +} BlockSummary; + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs); + /* ============================================================== * Private declarations * These prototypes shall only be called from within lib/compress @@ -12291,7 +17500,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); * Note: srcSizeHint == 0 means 0! */ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); /*! ZSTD_initCStream_internal() : * Private use only. Init streaming operation. @@ -12303,7 +17512,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); -void ZSTD_resetSeqStore(seqStore_t* ssPtr); +void ZSTD_resetSeqStore(SeqStore_t* ssPtr); /*! ZSTD_getCParamsFromCDict() : * as the name implies */ @@ -12342,11 +17551,10 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity); * This cannot be used when long range matching is enabled. * Zstd will use these sequences, and pass the literals to a secondary block * compressor. - * @return : An error code on failure. * NOTE: seqs are not verified! Invalid sequences can cause out-of-bounds memory * access and data corruption. */ -size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq); +void ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq); /** ZSTD_cycleLog() : * condition for correct operation : hashLog > 1 */ @@ -12357,30 +17565,89 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); */ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); +/* Returns 1 if an external sequence producer is registered, otherwise returns 0. */ +MEM_STATIC int ZSTD_hasExtSeqProd(const ZSTD_CCtx_params* params) { + return params->extSeqProdFunc != NULL; +} + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + #endif /* ZSTD_COMPRESS_H */ /**** ended inlining zstd_compress_internal.h ****/ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize, void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2); + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ /**** ended inlining zstd_compress_literals.h ****/ + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); switch(flSize) @@ -12399,16 +17666,30 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, } ZSTD_memcpy(ostart + flSize, src, srcSize); - DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); return srcSize + flSize; } +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); switch(flSize) { @@ -12426,27 +17707,51 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* } ostart[flSize] = *(const BYTE*)src; - DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); return flSize+1; } -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2) +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) { - size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); BYTE* const ostart = (BYTE*)dst; U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; + SymbolEncodingType_e hType = set_compressed; size_t cLitSize; - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", - disableLiteralCompression, (U32)srcSize); + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); /* Prepare nextEntropy assuming reusing the existing table */ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); @@ -12454,40 +17759,51 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, if (disableLiteralCompression) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); { HUF_repeat repeat = prevHuf->repeatMode; - int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? - HUF_compress1X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : - HUF_compress4X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); if (repeat != HUF_repeat_none) { /* reused the existing table */ - DEBUGLOG(5, "Reusing previous huffman table"); + DEBUGLOG(5, "reusing statistics from previous huffman block"); hType = set_repeat; } } - if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } if (cLitSize==1) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } if (hType == set_compressed) { /* using a newly constructed table */ @@ -12498,16 +17814,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, switch(lhSize) { case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); MEM_writeLE24(ostart, lhc); break; } case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); MEM_writeLE32(ostart, lhc); break; } case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); MEM_writeLE32(ostart, lhc); ostart[4] = (BYTE)(cLitSize >> 10); @@ -12522,7 +17841,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, /**** ended inlining compress/zstd_compress_literals.c ****/ /**** start inlining compress/zstd_compress_sequences.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12536,7 +17855,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ***************************************/ /**** start inlining zstd_compress_sequences.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12548,26 +17867,27 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, #ifndef ZSTD_COMPRESS_SEQUENCES_H #define ZSTD_COMPRESS_SEQUENCES_H +/**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: ../common/fse.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ typedef enum { ZSTD_defaultDisallowed = 0, ZSTD_defaultAllowed = 1 -} ZSTD_defaultPolicy_e; +} ZSTD_DefaultPolicy_e; -symbolEncodingType_e +SymbolEncodingType_e ZSTD_selectEncodingType( FSE_repeat* repeatMode, unsigned const* count, unsigned const max, size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, FSE_CTable const* prevCTable, short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_DefaultPolicy_e const isDefaultAllowed, ZSTD_strategy const strategy); size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type, unsigned* count, U32 max, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, @@ -12579,7 +17899,7 @@ size_t ZSTD_encodeSequences( FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); size_t ZSTD_fseBitCost( FSE_CTable const* ctable, @@ -12636,7 +17956,7 @@ static unsigned ZSTD_useLowProbCount(size_t const nbSeq) { /* Heuristic: This should cover most blocks <= 16K and * start to fade out after 16K to about 32K depending on - * comprssibility. + * compressibility. */ return nbSeq >= 2048; } @@ -12663,6 +17983,8 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t { unsigned cost = 0; unsigned s; + + assert(total > 0); for (s = 0; s <= max; ++s) { unsigned norm = (unsigned)((256 * count[s]) / total); if (count[s] != 0 && norm == 0) @@ -12729,20 +18051,20 @@ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, return cost >> 8; } -symbolEncodingType_e +SymbolEncodingType_e ZSTD_selectEncodingType( FSE_repeat* repeatMode, unsigned const* count, unsigned const max, size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, FSE_CTable const* prevCTable, short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_DefaultPolicy_e const isDefaultAllowed, ZSTD_strategy const strategy) { ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); if (mostFrequent == nbSeq) { *repeatMode = FSE_repeat_none; if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or less symbols, + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. * If basic encoding isn't possible, always choose RLE. */ @@ -12810,9 +18132,14 @@ ZSTD_selectEncodingType( return set_compressed; } +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type, unsigned* count, U32 max, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, @@ -12836,7 +18163,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ return 0; case set_compressed: { - S16 norm[MaxSeq + 1]; + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; size_t nbSeq_1 = nbSeq; const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); if (count[codeTable[nbSeq-1]] > 1) { @@ -12844,11 +18171,13 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, nbSeq_1--; } assert(nbSeq_1 > 1); - assert(entropyWorkspaceSize >= FSE_BUILD_CTABLE_WORKSPACE_SIZE(MaxSeq, MaxFSELog)); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), ""); - { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), ""); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); return NCountSize; } } @@ -12862,7 +18191,7 @@ ZSTD_encodeSequences_body( FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) + SeqDef const* sequences, size_t nbSeq, int longOffsets) { BIT_CStream_t blockStream; FSE_CState_t stateMatchLength; @@ -12882,19 +18211,19 @@ ZSTD_encodeSequences_body( FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); if (MEM_32bits()) BIT_flushBits(&blockStream); if (longOffsets) { U32 const ofBits = ofCodeTable[nbSeq-1]; unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); BIT_flushBits(&blockStream); } - BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, ofBits - extraBits); } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); } BIT_flushBits(&blockStream); @@ -12908,8 +18237,8 @@ ZSTD_encodeSequences_body( U32 const mlBits = ML_bits[mlCode]; DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", (unsigned)sequences[n].litLength, - (unsigned)sequences[n].matchLength + MINMATCH, - (unsigned)sequences[n].offset); + (unsigned)sequences[n].mlBase + MINMATCH, + (unsigned)sequences[n].offBase); /* 32b*/ /* 64b*/ /* (7)*/ /* (7)*/ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ @@ -12920,18 +18249,18 @@ ZSTD_encodeSequences_body( BIT_flushBits(&blockStream); /* (7)*/ BIT_addBits(&blockStream, sequences[n].litLength, llBits); if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); if (longOffsets) { unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); BIT_flushBits(&blockStream); /* (7)*/ } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, ofBits - extraBits); /* 31 */ } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ } BIT_flushBits(&blockStream); /* (7)*/ DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); @@ -12956,7 +18285,7 @@ ZSTD_encodeSequences_default( FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) + SeqDef const* sequences, size_t nbSeq, int longOffsets) { return ZSTD_encodeSequences_body(dst, dstCapacity, CTable_MatchLength, mlCodeTable, @@ -12968,13 +18297,13 @@ ZSTD_encodeSequences_default( #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_encodeSequences_bmi2( void* dst, size_t dstCapacity, FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) + SeqDef const* sequences, size_t nbSeq, int longOffsets) { return ZSTD_encodeSequences_body(dst, dstCapacity, CTable_MatchLength, mlCodeTable, @@ -12990,7 +18319,7 @@ size_t ZSTD_encodeSequences( FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) + SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) { DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); #if DYNAMIC_BMI2 @@ -13012,7 +18341,7 @@ size_t ZSTD_encodeSequences( /**** ended inlining compress/zstd_compress_sequences.c ****/ /**** start inlining compress/zstd_compress_superblock.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13026,7 +18355,7 @@ size_t ZSTD_encodeSequences( ***************************************/ /**** start inlining zstd_compress_superblock.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13065,284 +18394,6 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, /**** skipping file: zstd_compress_sequences.h ****/ /**** skipping file: zstd_compress_literals.h ****/ -/*-************************************* -* Superblock entropy buffer structs -***************************************/ -/** ZSTD_hufCTablesMetadata_t : - * Stores Literals Block Type for a super-block in hType, and - * huffman tree description in hufDesBuffer. - * hufDesSize refers to the size of huffman tree description in bytes. - * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */ -typedef struct { - symbolEncodingType_e hType; - BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; - size_t hufDesSize; -} ZSTD_hufCTablesMetadata_t; - -/** ZSTD_fseCTablesMetadata_t : - * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and - * fse tables in fseTablesBuffer. - * fseTablesSize refers to the size of fse tables in bytes. - * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */ -typedef struct { - symbolEncodingType_e llType; - symbolEncodingType_e ofType; - symbolEncodingType_e mlType; - BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; - size_t fseTablesSize; - size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ -} ZSTD_fseCTablesMetadata_t; - -typedef struct { - ZSTD_hufCTablesMetadata_t hufMetadata; - ZSTD_fseCTablesMetadata_t fseMetadata; -} ZSTD_entropyCTablesMetadata_t; - - -/** ZSTD_buildSuperBlockEntropy_literal() : - * Builds entropy for the super-block literals. - * Stores literals block type (raw, rle, compressed, repeat) and - * huffman description table to hufMetadata. - * @return : size of huffman description table or error code */ -static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize, - const ZSTD_hufCTables_t* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_hufCTablesMetadata_t* hufMetadata, - const int disableLiteralsCompression, - void* workspace, size_t wkspSize) -{ - BYTE* const wkspStart = (BYTE*)workspace; - BYTE* const wkspEnd = wkspStart + wkspSize; - BYTE* const countWkspStart = wkspStart; - unsigned* const countWksp = (unsigned*)workspace; - const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); - BYTE* const nodeWksp = countWkspStart + countWkspSize; - const size_t nodeWkspSize = wkspEnd-nodeWksp; - unsigned maxSymbolValue = 255; - unsigned huffLog = HUF_TABLELOG_DEFAULT; - HUF_repeat repeat = prevHuf->repeatMode; - - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); - - /* Prepare nextEntropy assuming reusing the existing table */ - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - - if (disableLiteralsCompression) { - DEBUGLOG(5, "set_basic - disabled"); - hufMetadata->hType = set_basic; - return 0; - } - - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) { - DEBUGLOG(5, "set_basic - too small"); - hufMetadata->hType = set_basic; - return 0; - } - } - - /* Scan input and build symbol stats */ - { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); - FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); - if (largest == srcSize) { - DEBUGLOG(5, "set_rle"); - hufMetadata->hType = set_rle; - return 0; - } - if (largest <= (srcSize >> 7)+4) { - DEBUGLOG(5, "set_basic - no gain"); - hufMetadata->hType = set_basic; - return 0; - } - } - - /* Validate the previous Huffman table */ - if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { - repeat = HUF_repeat_none; - } - - /* Build Huffman Tree */ - ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, - maxSymbolValue, huffLog, - nodeWksp, nodeWkspSize); - FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); - huffLog = (U32)maxBits; - { /* Build and write the CTable */ - size_t const newCSize = HUF_estimateCompressedSize( - (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); - size_t const hSize = HUF_writeCTable( - hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), - (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog); - /* Check against repeating the previous CTable */ - if (repeat != HUF_repeat_none) { - size_t const oldCSize = HUF_estimateCompressedSize( - (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); - if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { - DEBUGLOG(5, "set_repeat - smaller"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_repeat; - return 0; - } - } - if (newCSize + hSize >= srcSize) { - DEBUGLOG(5, "set_basic - no gains"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_basic; - return 0; - } - DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); - hufMetadata->hType = set_compressed; - nextHuf->repeatMode = HUF_repeat_check; - return hSize; - } - } -} - -/** ZSTD_buildSuperBlockEntropy_sequences() : - * Builds entropy for the super-block sequences. - * Stores symbol compression modes and fse table to fseMetadata. - * @return : size of fse tables or error code */ -static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr, - const ZSTD_fseCTables_t* prevEntropy, - ZSTD_fseCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize) -{ - BYTE* const wkspStart = (BYTE*)workspace; - BYTE* const wkspEnd = wkspStart + wkspSize; - BYTE* const countWkspStart = wkspStart; - unsigned* const countWksp = (unsigned*)workspace; - const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned); - BYTE* const cTableWksp = countWkspStart + countWkspSize; - const size_t cTableWkspSize = wkspEnd-cTableWksp; - ZSTD_strategy const strategy = cctxParams->cParams.strategy; - FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; - FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; - FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; - BYTE* const ostart = fseMetadata->fseTablesBuffer; - BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); - BYTE* op = ostart; - - assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE)); - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); - ZSTD_memset(workspace, 0, wkspSize); - - fseMetadata->lastCountSize = 0; - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - /* build CTable for Literal Lengths */ - { U32 LLtype; - unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - DEBUGLOG(5, "Building LL table"); - nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, - countWksp, max, mostFrequent, nbSeq, - LLFSELog, prevEntropy->litlengthCTable, - LL_defaultNorm, LL_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(set_basic < set_compressed && set_rle < set_compressed); - assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); - if (LLtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->llType = (symbolEncodingType_e) LLtype; - } } - /* build CTable for Offsets */ - { U32 Offtype; - unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; - DEBUGLOG(5, "Building OF table"); - nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, - countWksp, max, mostFrequent, nbSeq, - OffFSELog, prevEntropy->offcodeCTable, - OF_defaultNorm, OF_defaultNormLog, - defaultPolicy, strategy); - assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); - if (Offtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->ofType = (symbolEncodingType_e) Offtype; - } } - /* build CTable for MatchLengths */ - { U32 MLtype; - unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ - DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); - nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, - countWksp, max, mostFrequent, nbSeq, - MLFSELog, prevEntropy->matchlengthCTable, - ML_defaultNorm, ML_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), - cTableWksp, cTableWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); - if (MLtype == set_compressed) - fseMetadata->lastCountSize = countSize; - op += countSize; - fseMetadata->mlType = (symbolEncodingType_e) MLtype; - } } - assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer)); - return op-ostart; -} - - -/** ZSTD_buildSuperBlockEntropy() : - * Builds entropy for the super-block. - * @return : 0 on success or error code */ -static size_t -ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize) -{ - size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; - DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy"); - entropyMetadata->hufMetadata.hufDesSize = - ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize, - &prevEntropy->huf, &nextEntropy->huf, - &entropyMetadata->hufMetadata, - ZSTD_disableLiteralsCompression(cctxParams), - workspace, wkspSize); - FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed"); - entropyMetadata->fseMetadata.fseTablesSize = - ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr, - &prevEntropy->fse, &nextEntropy->fse, - cctxParams, - &entropyMetadata->fseMetadata, - workspace, wkspSize); - FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed"); - return 0; -} - /** ZSTD_compressSubBlock_literal() : * Compresses literals section for a sub-block. * When we have to write the Huffman table we will sometimes choose a header @@ -13360,13 +18411,14 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr, * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block * and the following sub-blocks' literals sections will be Treeless_Literals_Block. * @return : compressed size of literals section of a sub-block - * Or 0 if it unable to compress. + * Or 0 if unable to compress. * Or error code */ -static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - const BYTE* literals, size_t litSize, - void* dst, size_t dstSize, - const int bmi2, int writeEntropy, int* entropyWritten) +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) { size_t const header = writeEntropy ? 200 : 0; size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); @@ -13374,11 +18426,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, BYTE* const oend = ostart + dstSize; BYTE* op = ostart + lhSize; U32 const singleStream = lhSize == 3; - symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; size_t cLitSize = 0; - (void)bmi2; /* TODO bmi2... */ - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); *entropyWritten = 0; @@ -13400,9 +18450,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); } - /* TODO bmi2 */ - { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) - : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags); op += cSize; cLitSize += cSize; if (cSize == 0 || ERR_isError(cSize)) { @@ -13427,7 +18477,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, switch(lhSize) { case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); MEM_writeLE24(ostart, lhc); break; } @@ -13447,26 +18497,30 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, } *entropyWritten = 1; DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); - return op-ostart; + return (size_t)(op-ostart); } -static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { - const seqDef* const sstart = sequences; - const seqDef* const send = sequences + nbSeq; - const seqDef* sp = sstart; +static size_t +ZSTD_seqDecompressedSize(SeqStore_t const* seqStore, + const SeqDef* sequences, size_t nbSeqs, + size_t litSize, int lastSubBlock) +{ size_t matchLengthSum = 0; size_t litLengthSum = 0; - (void)litLengthSum; - while (send-sp > 0) { - ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + size_t n; + for (n=0; ncParams.windowLog > STREAM_ACCUMULATOR_MIN; BYTE* const ostart = (BYTE*)dst; @@ -13500,14 +18555,14 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables /* Sequences Header */ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, dstSize_tooSmall, ""); - if (nbSeq < 0x7F) + if (nbSeq < 128) *op++ = (BYTE)nbSeq; else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; if (nbSeq==0) { - return op - ostart; + return (size_t)(op - ostart); } /* seqHead : flags for FSE encoding type */ @@ -13529,7 +18584,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables } { size_t const bitstreamSize = ZSTD_encodeSequences( - op, oend - op, + op, (size_t)(oend - op), fseTables->matchlengthCTable, mlCode, fseTables->offcodeCTable, ofCode, fseTables->litlengthCTable, llCode, @@ -13573,7 +18628,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables #endif *entropyWritten = 1; - return op - ostart; + return (size_t)(op - ostart); } /** ZSTD_compressSubBlock() : @@ -13582,7 +18637,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables * Or 0 if it failed to compress. */ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - const seqDef* sequences, size_t nbSeq, + const SeqDef* sequences, size_t nbSeq, const BYTE* literals, size_t litSize, const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, const ZSTD_CCtx_params* cctxParams, @@ -13599,7 +18654,8 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, &entropyMetadata->hufMetadata, literals, litSize, - op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + op, (size_t)(oend-op), + bmi2, writeLitEntropy, litEntropyWritten); FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); if (cLitSize == 0) return 0; op += cLitSize; @@ -13609,18 +18665,18 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, sequences, nbSeq, llCode, mlCode, ofCode, cctxParams, - op, oend-op, + op, (size_t)(oend-op), bmi2, writeSeqEntropy, seqEntropyWritten); FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); if (cSeqSize == 0) return 0; op += cSeqSize; } /* Write block header */ - { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize; U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); MEM_writeLE24(ostart, cBlockHeader24); } - return op-ostart; + return (size_t)(op-ostart); } static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, @@ -13646,10 +18702,10 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit return 0; } -static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, +static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type, const BYTE* codeTable, unsigned maxCode, size_t nbSeq, const FSE_CTable* fseCTable, - const U32* additionalBits, + const U8* additionalBits, short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, void* workspace, size_t wkspSize) { @@ -13690,8 +18746,9 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, void* workspace, size_t wkspSize, int writeEntropy) { - size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, nbSeq, fseTables->offcodeCTable, NULL, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, @@ -13708,7 +18765,11 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, return cSeqSizeEstimate + sequencesSectionHeaderSize; } -static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, +typedef struct { + size_t estLitSize; + size_t estBlockSize; +} EstimatedBlockSize; +static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, const BYTE* ofCodeTable, const BYTE* llCodeTable, const BYTE* mlCodeTable, @@ -13716,15 +18777,17 @@ static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, const ZSTD_entropyCTables_t* entropy, const ZSTD_entropyCTablesMetadata_t* entropyMetadata, void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy) { - size_t cSizeEstimate = 0; - cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, - &entropy->huf, &entropyMetadata->hufMetadata, - workspace, wkspSize, writeLitEntropy); - cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + int writeLitEntropy, int writeSeqEntropy) +{ + EstimatedBlockSize ebs; + ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, workspace, wkspSize, writeSeqEntropy); - return cSizeEstimate + ZSTD_blockHeaderSize; + ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize; + return ebs; } static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) @@ -13738,14 +18801,57 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe return 0; } +static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount) +{ + size_t n, total = 0; + assert(sp != NULL); + for (n=0; n %zu bytes", seqCount, (const void*)sp, total); + return total; +} + +#define BYTESCALE 256 + +static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs, + size_t targetBudget, size_t avgLitCost, size_t avgSeqCost, + int firstSubBlock) +{ + size_t n, budget = 0, inSize=0; + /* entropy headers */ + size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */ + assert(firstSubBlock==0 || firstSubBlock==1); + budget += headerSize; + + /* first sequence => at least one sequence*/ + budget += sp[0].litLength * avgLitCost + avgSeqCost; + if (budget > targetBudget) return 1; + inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH); + + /* loop over sequences */ + for (n=1; n targetBudget) + /* though continue to expand until the sub-block is deemed compressible */ + && (budget < inSize * BYTESCALE) ) + break; + } + + return n; +} + /** ZSTD_compressSubBlock_multi() : * Breaks super-block into multiple sub-blocks and compresses them. - * Entropy will be written to the first block. - * The following blocks will use repeat mode to compress. - * All sub-blocks are compressed blocks (no raw or rle blocks). - * @return : compressed size of the super block (which is multiple ZSTD blocks) - * Or 0 if it failed to compress. */ -static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + * Entropy will be written into the first block. + * The following blocks use repeat_mode to compress. + * Sub-blocks are all compressed, except the last one when beneficial. + * @return : compressed size of the super block (which features multiple ZSTD blocks) + * or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr, const ZSTD_compressedBlockState_t* prevCBlock, ZSTD_compressedBlockState_t* nextCBlock, const ZSTD_entropyCTablesMetadata_t* entropyMetadata, @@ -13755,12 +18861,14 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, const int bmi2, U32 lastBlock, void* workspace, size_t wkspSize) { - const seqDef* const sstart = seqStorePtr->sequencesStart; - const seqDef* const send = seqStorePtr->sequences; - const seqDef* sp = sstart; + const SeqDef* const sstart = seqStorePtr->sequencesStart; + const SeqDef* const send = seqStorePtr->sequences; + const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */ + size_t const nbSeqs = (size_t)(send - sstart); const BYTE* const lstart = seqStorePtr->litStart; const BYTE* const lend = seqStorePtr->lit; const BYTE* lp = lstart; + size_t const nbLiterals = (size_t)(lend - lstart); BYTE const* ip = (BYTE const*)src; BYTE const* const iend = ip + srcSize; BYTE* const ostart = (BYTE*)dst; @@ -13769,120 +18877,179 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, const BYTE* llCodePtr = seqStorePtr->llCode; const BYTE* mlCodePtr = seqStorePtr->mlCode; const BYTE* ofCodePtr = seqStorePtr->ofCode; - size_t targetCBlockSize = cctxParams->targetCBlockSize; - size_t litSize, seqCount; - int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */ + size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize); + int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed); int writeSeqEntropy = 1; - int lastSequence = 0; - DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", - (unsigned)(lend-lp), (unsigned)(send-sstart)); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)", + (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart)); - litSize = 0; - seqCount = 0; - do { - size_t cBlockSizeEstimate = 0; - if (sstart == send) { - lastSequence = 1; - } else { - const seqDef* const sequence = sp + seqCount; - lastSequence = sequence == send - 1; - litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; - seqCount++; + /* let's start by a general estimation for the full block */ + if (nbSeqs > 0) { + EstimatedBlockSize const ebs = + ZSTD_estimateSubBlockSize(lp, nbLiterals, + ofCodePtr, llCodePtr, mlCodePtr, nbSeqs, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, + writeLitEntropy, writeSeqEntropy); + /* quick estimation */ + size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE; + size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs; + const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1); + size_t n, avgBlockBudget, blockBudgetSupp=0; + avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks; + DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes", + (unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE, + (unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE); + /* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately + * this will result in the production of a single uncompressed block covering @srcSize.*/ + if (ebs.estBlockSize > srcSize) return 0; + + /* compress and write sub-blocks */ + assert(nbSubBlocks>0); + for (n=0; n < nbSubBlocks-1; n++) { + /* determine nb of sequences for current sub-block + nbLiterals from next sequence */ + size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp), + avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0); + /* if reached last sequence : break to last sub-block (simplification) */ + assert(seqCount <= (size_t)(send-sp)); + if (sp + seqCount == send) break; + assert(seqCount > 0); + /* compress sub-block */ + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = countLiterals(seqStorePtr, sp, seqCount); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + 0); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* check compressibility, update state components */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; + blockBudgetSupp = 0; + } } + /* otherwise : do not compress yet, coalesce current sub-block with following one */ } - if (lastSequence) { - assert(lp <= lend); - assert(litSize <= (size_t)(lend - lp)); - litSize = (size_t)(lend - lp); - } - /* I think there is an optimization opportunity here. - * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful - * since it recalculates estimate from scratch. - * For example, it would recount literal distribution and symbol codes everytime. - */ - cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, - &nextCBlock->entropy, entropyMetadata, - workspace, wkspSize, writeLitEntropy, writeSeqEntropy); - if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { - int litEntropyWritten = 0; - int seqEntropyWritten = 0; - const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); - const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, - sp, seqCount, - lp, litSize, - llCodePtr, mlCodePtr, ofCodePtr, - cctxParams, - op, oend-op, - bmi2, writeLitEntropy, writeSeqEntropy, - &litEntropyWritten, &seqEntropyWritten, - lastBlock && lastSequence); - FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); - if (cSize > 0 && cSize < decompressedSize) { - DEBUGLOG(5, "Committed the sub-block"); - assert(ip + decompressedSize <= iend); - ip += decompressedSize; - sp += seqCount; - lp += litSize; - op += cSize; - llCodePtr += seqCount; - mlCodePtr += seqCount; - ofCodePtr += seqCount; - litSize = 0; - seqCount = 0; - /* Entropy only needs to be written once */ - if (litEntropyWritten) { - writeLitEntropy = 0; - } - if (seqEntropyWritten) { - writeSeqEntropy = 0; - } + } /* if (nbSeqs > 0) */ + + /* write last block */ + DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp)); + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = (size_t)(lend - lp); + size_t seqCount = (size_t)(send - sp); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* update pointers, the nb of literals borrowed from next sequence must be preserved */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; } - } while (!lastSequence); + } + + if (writeLitEntropy) { - DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + DEBUGLOG(5, "Literal entropy tables were never written"); ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); } if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { /* If we haven't written our entropy tables, then we've violated our contract and * must emit an uncompressed block. */ - DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block"); return 0; } + if (ip < iend) { - size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); - DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + /* some data left : last part of the block sent uncompressed */ + size_t const rSize = (size_t)((iend - ip)); + size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock); + DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize)); FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); assert(cSize != 0); op += cSize; /* We have to regenerate the repcodes because we've skipped some sequences */ if (sp < send) { - seqDef const* seq; - repcodes_t rep; + const SeqDef* seq; + Repcodes_t rep; ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); for (seq = sstart; seq < sp; ++seq) { - rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); } ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); } } - DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); - return op-ostart; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u", + (unsigned)(op-ostart)); + return (size_t)(op-ostart); } size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, - void const* src, size_t srcSize, - unsigned lastBlock) { + const void* src, size_t srcSize, + unsigned lastBlock) +{ ZSTD_entropyCTablesMetadata_t entropyMetadata; - FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore, + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, &entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), ""); return ZSTD_compressSubBlock_multi(&zc->seqStore, zc->blockState.prevCBlock, @@ -13892,12 +19059,252 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, dst, dstCapacity, src, srcSize, zc->bmi2, lastBlock, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); } /**** ended inlining compress/zstd_compress_superblock.c ****/ +/**** start inlining compress/zstd_preSplit.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: hist.h ****/ +/**** skipping file: zstd_preSplit.h ****/ + + +#define BLOCKSIZE_MIN 3500 +#define THRESHOLD_PENALTY_RATE 16 +#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2) +#define THRESHOLD_PENALTY 3 + +#define HASHLENGTH 2 +#define HASHLOG_MAX 10 +#define HASHTABLESIZE (1 << HASHLOG_MAX) +#define HASHMASK (HASHTABLESIZE - 1) +#define KNUTH 0x9e3779b9 + +/* for hashLog > 8, hash 2 bytes. + * for hashLog == 8, just take the byte, no hashing. + * The speed of this method relies on compile-time constant propagation */ +FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog) +{ + assert(hashLog >= 8); + if (hashLog == 8) return (U32)((const BYTE*)p)[0]; + assert(hashLog <= HASHLOG_MAX); + return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog); +} + + +typedef struct { + unsigned events[HASHTABLESIZE]; + size_t nbEvents; +} Fingerprint; +typedef struct { + Fingerprint pastEvents; + Fingerprint newEvents; +} FPStats; + +static void initStats(FPStats* fpstats) +{ + ZSTD_memset(fpstats, 0, sizeof(FPStats)); +} + +FORCE_INLINE_TEMPLATE void +addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) +{ + const char* p = (const char*)src; + size_t limit = srcSize - HASHLENGTH + 1; + size_t n; + assert(srcSize >= HASHLENGTH); + for (n = 0; n < limit; n+=samplingRate) { + fp->events[hash2(p+n, hashLog)]++; + } + fp->nbEvents += limit/samplingRate; +} + +FORCE_INLINE_TEMPLATE void +recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) +{ + ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog)); + fp->nbEvents = 0; + addEvents_generic(fp, src, srcSize, samplingRate, hashLog); +} + +typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize); + +#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate + +#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \ + static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \ + { \ + recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \ + } + +ZSTD_GEN_RECORD_FINGERPRINT(1, 10) +ZSTD_GEN_RECORD_FINGERPRINT(5, 10) +ZSTD_GEN_RECORD_FINGERPRINT(11, 9) +ZSTD_GEN_RECORD_FINGERPRINT(43, 8) + + +static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); } + +static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog) +{ + U64 distance = 0; + size_t n; + assert(hashLog <= HASHLOG_MAX); + for (n = 0; n < ((size_t)1 << hashLog); n++) { + distance += + abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents); + } + return distance; +} + +/* Compare newEvents with pastEvents + * return 1 when considered "too different" + */ +static int compareFingerprints(const Fingerprint* ref, + const Fingerprint* newfp, + int penalty, + unsigned hashLog) +{ + assert(ref->nbEvents > 0); + assert(newfp->nbEvents > 0); + { U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents; + U64 deviation = fpDistance(ref, newfp, hashLog); + U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; + return deviation >= threshold; + } +} + +static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + acc->events[n] += newfp->events[n]; + } + acc->nbEvents += newfp->nbEvents; +} + +static void flushEvents(FPStats* fpstats) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + fpstats->pastEvents.events[n] = fpstats->newEvents.events[n]; + } + fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents; + ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents)); +} + +static void removeEvents(Fingerprint* acc, const Fingerprint* slice) +{ + size_t n; + for (n = 0; n < HASHTABLESIZE; n++) { + assert(acc->events[n] >= slice->events[n]); + acc->events[n] -= slice->events[n]; + } + acc->nbEvents -= slice->nbEvents; +} + +#define CHUNKSIZE (8 << 10) +static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize) +{ + static const RecordEvents_f records_fs[] = { + FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1) + }; + static const unsigned hashParams[] = { 8, 9, 10, 10 }; + const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]); + FPStats* const fpstats = (FPStats*)workspace; + const char* p = (const char*)blockStart; + int penalty = THRESHOLD_PENALTY; + size_t pos = 0; + assert(blockSize == (128 << 10)); + assert(workspace != NULL); + assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); + ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); + assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; + + initStats(fpstats); + record_f(&fpstats->pastEvents, p, CHUNKSIZE); + for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) { + record_f(&fpstats->newEvents, p + pos, CHUNKSIZE); + if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) { + return pos; + } else { + mergeEvents(&fpstats->pastEvents, &fpstats->newEvents); + if (penalty > 0) penalty--; + } + } + assert(pos == blockSize); + return blockSize; + (void)flushEvents; (void)removeEvents; +} + +/* ZSTD_splitBlock_fromBorders(): very fast strategy : + * compare fingerprint from beginning and end of the block, + * derive from their difference if it's preferable to split in the middle, + * repeat the process a second time, for finer grained decision. + * 3 times did not brought improvements, so I stopped at 2. + * Benefits are good enough for a cheap heuristic. + * More accurate splitting saves more, but speed impact is also more perceptible. + * For better accuracy, use more elaborate variant *_byChunks. + */ +static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize, + void* workspace, size_t wkspSize) +{ +#define SEGMENT_SIZE 512 + FPStats* const fpstats = (FPStats*)workspace; + Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned)); + assert(blockSize == (128 << 10)); + assert(workspace != NULL); + assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); + ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); + assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; + + initStats(fpstats); + HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE); + HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE); + fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE; + if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8)) + return blockSize; + + HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE); + middleEvents->nbEvents = SEGMENT_SIZE; + { U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8); + U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8); + U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3; + if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance) + return 64 KB; + return (distFromBegin > distFromEnd) ? 32 KB : 96 KB; + } +} + +size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, + int level, + void* workspace, size_t wkspSize) +{ + DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level); + assert(0<=level && level<=4); + if (level == 0) + return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize); + /* level >= 1*/ + return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize); +} +/**** ended inlining compress/zstd_preSplit.c ****/ /**** start inlining compress/zstd_compress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13909,235 +19316,20 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, /*-************************************* * Dependencies ***************************************/ +/**** skipping file: ../common/allocations.h ****/ /**** skipping file: ../common/zstd_deps.h ****/ -/**** start inlining ../common/cpu.h ****/ -/* - * Copyright (c) 2018-2021, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_COMMON_CPU_H -#define ZSTD_COMMON_CPU_H - -/** - * Implementation taken from folly/CpuId.h - * https://github.com/facebook/folly/blob/master/folly/CpuId.h - */ - -/**** skipping file: mem.h ****/ - -#ifdef _MSC_VER -#include -#endif - -typedef struct { - U32 f1c; - U32 f1d; - U32 f7b; - U32 f7c; -} ZSTD_cpuid_t; - -MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { - U32 f1c = 0; - U32 f1d = 0; - U32 f7b = 0; - U32 f7c = 0; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - int reg[4]; - __cpuid((int*)reg, 0); - { - int const n = reg[0]; - if (n >= 1) { - __cpuid((int*)reg, 1); - f1c = (U32)reg[2]; - f1d = (U32)reg[3]; - } - if (n >= 7) { - __cpuidex((int*)reg, 7, 0); - f7b = (U32)reg[1]; - f7c = (U32)reg[2]; - } - } -#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) - /* The following block like the normal cpuid branch below, but gcc - * reserves ebx for use of its pic register so we must specially - * handle the save and restore to avoid clobbering the register - */ - U32 n; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(n) - : "a"(0) - : "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "popl %%ebx\n\t" - : "=a"(f1a), "=c"(f1c), "=d"(f1d) - : "a"(1)); - } - if (n >= 7) { - __asm__( - "pushl %%ebx\n\t" - "cpuid\n\t" - "movl %%ebx, %%eax\n\t" - "popl %%ebx" - : "=a"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } -#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) - U32 n; - __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); - if (n >= 1) { - U32 f1a; - __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); - } - if (n >= 7) { - U32 f7a; - __asm__("cpuid" - : "=a"(f7a), "=b"(f7b), "=c"(f7c) - : "a"(7), "c"(0) - : "edx"); - } -#endif - { - ZSTD_cpuid_t cpuid; - cpuid.f1c = f1c; - cpuid.f1d = f1d; - cpuid.f7b = f7b; - cpuid.f7c = f7c; - return cpuid; - } -} - -#define X(name, r, bit) \ - MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ - return ((cpuid.r) & (1U << bit)) != 0; \ - } - -/* cpuid(1): Processor Info and Feature Bits. */ -#define C(name, bit) X(name, f1c, bit) - C(sse3, 0) - C(pclmuldq, 1) - C(dtes64, 2) - C(monitor, 3) - C(dscpl, 4) - C(vmx, 5) - C(smx, 6) - C(eist, 7) - C(tm2, 8) - C(ssse3, 9) - C(cnxtid, 10) - C(fma, 12) - C(cx16, 13) - C(xtpr, 14) - C(pdcm, 15) - C(pcid, 17) - C(dca, 18) - C(sse41, 19) - C(sse42, 20) - C(x2apic, 21) - C(movbe, 22) - C(popcnt, 23) - C(tscdeadline, 24) - C(aes, 25) - C(xsave, 26) - C(osxsave, 27) - C(avx, 28) - C(f16c, 29) - C(rdrand, 30) -#undef C -#define D(name, bit) X(name, f1d, bit) - D(fpu, 0) - D(vme, 1) - D(de, 2) - D(pse, 3) - D(tsc, 4) - D(msr, 5) - D(pae, 6) - D(mce, 7) - D(cx8, 8) - D(apic, 9) - D(sep, 11) - D(mtrr, 12) - D(pge, 13) - D(mca, 14) - D(cmov, 15) - D(pat, 16) - D(pse36, 17) - D(psn, 18) - D(clfsh, 19) - D(ds, 21) - D(acpi, 22) - D(mmx, 23) - D(fxsr, 24) - D(sse, 25) - D(sse2, 26) - D(ss, 27) - D(htt, 28) - D(tm, 29) - D(pbe, 31) -#undef D - -/* cpuid(7): Extended Features. */ -#define B(name, bit) X(name, f7b, bit) - B(bmi1, 3) - B(hle, 4) - B(avx2, 5) - B(smep, 7) - B(bmi2, 8) - B(erms, 9) - B(invpcid, 10) - B(rtm, 11) - B(mpx, 14) - B(avx512f, 16) - B(avx512dq, 17) - B(rdseed, 18) - B(adx, 19) - B(smap, 20) - B(avx512ifma, 21) - B(pcommit, 22) - B(clflushopt, 23) - B(clwb, 24) - B(avx512pf, 26) - B(avx512er, 27) - B(avx512cd, 28) - B(sha, 29) - B(avx512bw, 30) - B(avx512vl, 31) -#undef B -#define C(name, bit) X(name, f7c, bit) - C(prefetchwt1, 0) - C(avx512vbmi, 1) -#undef C - -#undef X - -#endif /* ZSTD_COMMON_CPU_H */ -/**** ended inlining ../common/cpu.h ****/ /**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ +/**** skipping file: ../common/error_private.h ****/ /**** skipping file: hist.h ****/ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_compress_sequences.h ****/ /**** skipping file: zstd_compress_literals.h ****/ /**** start inlining zstd_fast.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14149,34 +19341,27 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { #ifndef ZSTD_FAST_H #define ZSTD_FAST_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: ../common/mem.h ****/ /**** skipping file: zstd_compress_internal.h ****/ -void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); +void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -#if defined (__cplusplus) -} -#endif - #endif /* ZSTD_FAST_H */ /**** ended inlining zstd_fast.h ****/ /**** start inlining zstd_double_fast.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14188,35 +19373,39 @@ size_t ZSTD_compressBlock_fast_extDict( #ifndef ZSTD_DOUBLE_FAST_H #define ZSTD_DOUBLE_FAST_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: ../common/mem.h ****/ /**** skipping file: zstd_compress_internal.h ****/ -void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + +void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); + size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_doubleFast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); - -#if defined (__cplusplus) -} -#endif +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict +#else +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ #endif /* ZSTD_DOUBLE_FAST_H */ /**** ended inlining zstd_double_fast.h ****/ /**** start inlining zstd_lazy.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14228,10 +19417,6 @@ size_t ZSTD_compressBlock_doubleFast_extDict( #ifndef ZSTD_LAZY_H #define ZSTD_LAZY_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: zstd_compress_internal.h ****/ /** @@ -14242,70 +19427,180 @@ extern "C" { */ #define ZSTD_LAZY_DDSS_BUCKET_LOG 2 -U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ -void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) +U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip); void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ +#endif -size_t ZSTD_compressBlock_btlazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_greedy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); - -size_t ZSTD_compressBlock_btlazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +size_t ZSTD_compressBlock_greedy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_greedy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); - -size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -size_t ZSTD_compressBlock_greedy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW ZSTD_compressBlock_greedy_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE ZSTD_compressBlock_greedy_dictMatchState +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW ZSTD_compressBlock_greedy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH ZSTD_compressBlock_greedy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_greedy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT ZSTD_compressBlock_greedy_extDict +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW ZSTD_compressBlock_greedy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_GREEDY NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_lazy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -size_t ZSTD_compressBlock_lazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btlazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -#if defined (__cplusplus) -} +#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy +#define ZSTD_COMPRESSBLOCK_LAZY_ROW ZSTD_compressBlock_lazy_row +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE ZSTD_compressBlock_lazy_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT ZSTD_compressBlock_lazy_extDict +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW ZSTD_compressBlock_lazy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY NULL +#define ZSTD_COMPRESSBLOCK_LAZY_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2 +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW ZSTD_compressBlock_lazy2_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE ZSTD_compressBlock_lazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy2_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy2_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy2_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT ZSTD_compressBlock_lazy2_extDict +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW ZSTD_compressBlock_lazy2_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY2 NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2 +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE ZSTD_compressBlock_btlazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT ZSTD_compressBlock_btlazy2_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTLAZY2 NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL #endif #endif /* ZSTD_LAZY_H */ /**** ended inlining zstd_lazy.h ****/ /**** start inlining zstd_opt.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14317,53 +19612,69 @@ size_t ZSTD_compressBlock_btlazy2_extDict( #ifndef ZSTD_OPT_H #define ZSTD_OPT_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: zstd_compress_internal.h ****/ +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) /* used in ZSTD_loadDictionaryContent() */ -void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); +void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend); +#endif +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btopt( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); -size_t ZSTD_compressBlock_btultra2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); - - size_t ZSTD_compressBlock_btopt_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE ZSTD_compressBlock_btopt_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT ZSTD_compressBlock_btopt_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTOPT NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_btultra_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize); - -size_t ZSTD_compressBlock_btopt_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); size_t ZSTD_compressBlock_btultra_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); /* note : no btultra2 variant for extDict nor dictMatchState, * because btultra2 is not meant to work with dictionaries * and is only specific for the first block (no prefix) */ +size_t ZSTD_compressBlock_btultra2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); -#if defined (__cplusplus) -} +#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE ZSTD_compressBlock_btultra_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT ZSTD_compressBlock_btultra_extDict +#define ZSTD_COMPRESSBLOCK_BTULTRA2 ZSTD_compressBlock_btultra2 +#else +#define ZSTD_COMPRESSBLOCK_BTULTRA NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL #endif #endif /* ZSTD_OPT_H */ /**** ended inlining zstd_opt.h ****/ /**** start inlining zstd_ldm.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14375,10 +19686,6 @@ size_t ZSTD_compressBlock_btultra_extDict( #ifndef ZSTD_LDM_H #define ZSTD_LDM_H -#if defined (__cplusplus) -extern "C" { -#endif - /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: ../zstd.h ****/ @@ -14407,7 +19714,7 @@ void ZSTD_ldm_fillHashTable( * sequences. */ size_t ZSTD_ldm_generateSequences( - ldmState_t* ldms, rawSeqStore_t* sequences, + ldmState_t* ldms, RawSeqStore_t* sequences, ldmParams_t const* params, void const* src, size_t srcSize); /** @@ -14428,8 +19735,9 @@ size_t ZSTD_ldm_generateSequences( * two. We handle that case correctly, and update `rawSeqStore` appropriately. * NOTE: This function does not return any errors. */ -size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore, + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_ParamSwitch_e useRowMatchFinder, void const* src, size_t srcSize); /** @@ -14439,7 +19747,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, * Avoids emitting matches less than `minMatch` bytes. * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). */ -void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, +void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch); /* ZSTD_ldm_skipRawSeqStoreBytes(): @@ -14447,7 +19755,7 @@ void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, * Not to be used in conjunction with ZSTD_ldm_skipSequences(). * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). */ -void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); +void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes); /** ZSTD_ldm_getTableSize() : * Estimate the space needed for long distance matching tables or 0 if LDM is @@ -14473,13 +19781,10 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_compressionParameters const* cParams); -#if defined (__cplusplus) -} -#endif - #endif /* ZSTD_FAST_H */ /**** ended inlining zstd_ldm.h ****/ /**** skipping file: zstd_compress_superblock.h ****/ +/**** skipping file: ../common/bits.h ****/ /* *************************************************************** * Tuning parameters @@ -14494,19 +19799,34 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, # define ZSTD_COMPRESS_HEAPMODE 0 #endif +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_MatchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif /*-************************************* * Helper functions ***************************************/ /* ZSTD_compressBound() - * Note that the result from this function is only compatible with the "normal" - * full-block strategy. - * When there are a lot of small blocks due to frequent flush in streaming mode - * the overhead of headers can make the compressed data to be larger than the - * return value of ZSTD_compressBound(). + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). */ size_t ZSTD_compressBound(size_t srcSize) { - return ZSTD_COMPRESSBOUND(srcSize); + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; } @@ -14519,11 +19839,15 @@ struct ZSTD_CDict_s { ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ ZSTD_cwksp workspace; - ZSTD_matchState_t matchState; + ZSTD_MatchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_ParamSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -14536,7 +19860,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) assert(cctx != NULL); ZSTD_memset(cctx, 0, sizeof(*cctx)); cctx->customMem = memManager; - cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); assert(!ZSTD_isError(err)); (void)err; @@ -14570,11 +19894,12 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; - /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + /* statically sized space. tmpWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); + cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -14611,15 +19936,13 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { + DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - { - int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); ZSTD_freeCCtxContent(cctx); - if (!cctxInWorkspace) { - ZSTD_customFree(cctx, cctx->customMem); - } + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); } return 0; } @@ -14652,14 +19975,85 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) } /* private API call, for dictBuilder only */ -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } +const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } -/* Returns 1 if compression parameters are such that we should - * enable long distance matching (wlog >= 27, strategy >= btopt). - * Returns 0 otherwise. +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. */ -static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) { - return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} + +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + return mode; +} + +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} + +/* Returns ZSTD_ps_enable if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns ZSTD_ps_disable otherwise. + */ +static ZSTD_ParamSwitch_e ZSTD_resolveEnableLdm(ZSTD_ParamSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} + +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + +static ZSTD_ParamSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_ParamSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} + +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; } static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( @@ -14670,15 +20064,19 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); cctxParams.cParams = cParams; - if (ZSTD_CParams_shouldEnableLdm(&cParams)) { - DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params"); - cctxParams.ldmParams.enableLdm = 1; - /* LDM is enabled by default for optimal parser and window size >= 128MB */ + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); assert(cctxParams.ldmParams.hashRateLog < 32); } - + cctxParams.postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.postBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); assert(!ZSTD_checkCParams(cParams)); return cctxParams; } @@ -14724,10 +20122,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) #define ZSTD_NO_CLEVEL 0 /** - * Initializes the cctxParams from params and compressionLevel. + * Initializes `cctxParams` from `params` and `compressionLevel`. * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. */ -static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) { assert(!ZSTD_checkCParams(params->cParams)); ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); @@ -14737,6 +20138,14 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par * But, set it for tracing anyway. */ cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->postBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->postBlockSplitter, cctxParams->ldmParams.enableLdm); } size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) @@ -14749,7 +20158,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete /** * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. - * @param param Validated zstd parameters. + * @param params Validated zstd parameters. */ static void ZSTD_CCtxParams_setZstdParams( ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) @@ -14858,8 +20267,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_enableLongDistanceMatching: - bounds.lowerBound = 0; - bounds.upperBound = 1; + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; return bounds; case ZSTD_c_ldmHashLog: @@ -14906,9 +20315,9 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_literalCompressionMode: - ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); - bounds.lowerBound = ZSTD_lcm_auto; - bounds.upperBound = ZSTD_lcm_uncompressed; + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; return bounds; case ZSTD_c_targetCBlockSize: @@ -14937,6 +20346,46 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = 1; return bounds; + case ZSTD_c_splitAfterSequences: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_blockSplitterLevel: + bounds.lowerBound = 0; + bounds.upperBound = ZSTD_BLOCKSPLITTER_LEVEL_MAX; + return bounds; + + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_repcodeResolution: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -14955,10 +20404,11 @@ static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) return 0; } -#define BOUNDCHECK(cParam, val) { \ - RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ - parameter_outOfBound, "Param out of bounds"); \ -} +#define BOUNDCHECK(cParam, val) \ + do { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ + } while (0) static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) @@ -14972,6 +20422,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_minMatch: case ZSTD_c_targetLength: case ZSTD_c_strategy: + case ZSTD_c_blockSplitterLevel: return 1; case ZSTD_c_format: @@ -14998,6 +20449,13 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_stableOutBuffer: case ZSTD_c_blockDelimiters: case ZSTD_c_validateSequences: + case ZSTD_c_splitAfterSequences: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_repcodeResolution: default: return 0; } @@ -15010,7 +20468,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) if (ZSTD_isUpdateAuthorized(param)) { cctx->cParamsChanged = 1; } else { - RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); } } switch(param) @@ -15050,6 +20508,14 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_stableOutBuffer: case ZSTD_c_blockDelimiters: case ZSTD_c_validateSequences: + case ZSTD_c_splitAfterSequences: + case ZSTD_c_blockSplitterLevel: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_repcodeResolution: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -15105,12 +20571,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_minMatch : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_minMatch, value); - CCtxParams->cParams.minMatch = value; + CCtxParams->cParams.minMatch = (U32)value; return CCtxParams->cParams.minMatch; case ZSTD_c_targetLength : BOUNDCHECK(ZSTD_c_targetLength, value); - CCtxParams->cParams.targetLength = value; + CCtxParams->cParams.targetLength = (U32)value; return CCtxParams->cParams.targetLength; case ZSTD_c_strategy : @@ -15123,12 +20589,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, /* Content size written in frame header _when known_ (default:1) */ DEBUGLOG(4, "set content size flag = %u", (value!=0)); CCtxParams->fParams.contentSizeFlag = value != 0; - return CCtxParams->fParams.contentSizeFlag; + return (size_t)CCtxParams->fParams.contentSizeFlag; case ZSTD_c_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ CCtxParams->fParams.checksumFlag = value != 0; - return CCtxParams->fParams.checksumFlag; + return (size_t)CCtxParams->fParams.checksumFlag; case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); @@ -15137,18 +20603,18 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_forceMaxWindow : CCtxParams->forceWindow = (value != 0); - return CCtxParams->forceWindow; + return (size_t)CCtxParams->forceWindow; case ZSTD_c_forceAttachDict : { const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; - BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); CCtxParams->attachDictPref = pref; return CCtxParams->attachDictPref; } case ZSTD_c_literalCompressionMode : { - const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; - BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); + const ZSTD_ParamSwitch_e lcm = (ZSTD_ParamSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); CCtxParams->literalCompressionMode = lcm; return CCtxParams->literalCompressionMode; } @@ -15160,7 +20626,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, #else FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); CCtxParams->nbWorkers = value; - return CCtxParams->nbWorkers; + return (size_t)(CCtxParams->nbWorkers); #endif case ZSTD_c_jobSize : @@ -15173,7 +20639,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, value = ZSTDMT_JOBSIZE_MIN; FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); assert(value >= 0); - CCtxParams->jobSize = value; + CCtxParams->jobSize = (size_t)value; return CCtxParams->jobSize; #endif @@ -15184,7 +20650,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, #else FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); CCtxParams->overlapLog = value; - return CCtxParams->overlapLog; + return (size_t)CCtxParams->overlapLog; #endif case ZSTD_c_rsyncable : @@ -15194,52 +20660,55 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, #else FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); CCtxParams->rsyncable = value; - return CCtxParams->rsyncable; + return (size_t)CCtxParams->rsyncable; #endif case ZSTD_c_enableDedicatedDictSearch : CCtxParams->enableDedicatedDictSearch = (value!=0); - return CCtxParams->enableDedicatedDictSearch; + return (size_t)CCtxParams->enableDedicatedDictSearch; case ZSTD_c_enableLongDistanceMatching : - CCtxParams->ldmParams.enableLdm = (value!=0); + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); + CCtxParams->ldmParams.enableLdm = (ZSTD_ParamSwitch_e)value; return CCtxParams->ldmParams.enableLdm; case ZSTD_c_ldmHashLog : if (value!=0) /* 0 ==> auto */ BOUNDCHECK(ZSTD_c_ldmHashLog, value); - CCtxParams->ldmParams.hashLog = value; + CCtxParams->ldmParams.hashLog = (U32)value; return CCtxParams->ldmParams.hashLog; case ZSTD_c_ldmMinMatch : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_ldmMinMatch, value); - CCtxParams->ldmParams.minMatchLength = value; + CCtxParams->ldmParams.minMatchLength = (U32)value; return CCtxParams->ldmParams.minMatchLength; case ZSTD_c_ldmBucketSizeLog : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); - CCtxParams->ldmParams.bucketSizeLog = value; + CCtxParams->ldmParams.bucketSizeLog = (U32)value; return CCtxParams->ldmParams.bucketSizeLog; case ZSTD_c_ldmHashRateLog : - RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, - parameter_outOfBound, "Param out of bounds!"); - CCtxParams->ldmParams.hashRateLog = value; + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = (U32)value; return CCtxParams->ldmParams.hashRateLog; case ZSTD_c_targetCBlockSize : - if (value!=0) /* 0 ==> default */ + if (value!=0) { /* 0 ==> default */ + value = MAX(value, ZSTD_TARGETCBLOCKSIZE_MIN); BOUNDCHECK(ZSTD_c_targetCBlockSize, value); - CCtxParams->targetCBlockSize = value; + } + CCtxParams->targetCBlockSize = (U32)value; return CCtxParams->targetCBlockSize; case ZSTD_c_srcSizeHint : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_srcSizeHint, value); CCtxParams->srcSizeHint = value; - return CCtxParams->srcSizeHint; + return (size_t)CCtxParams->srcSizeHint; case ZSTD_c_stableInBuffer: BOUNDCHECK(ZSTD_c_stableInBuffer, value); @@ -15253,13 +20722,55 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_blockDelimiters: BOUNDCHECK(ZSTD_c_blockDelimiters, value); - CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + CCtxParams->blockDelimiters = (ZSTD_SequenceFormat_e)value; return CCtxParams->blockDelimiters; case ZSTD_c_validateSequences: BOUNDCHECK(ZSTD_c_validateSequences, value); CCtxParams->validateSequences = value; - return CCtxParams->validateSequences; + return (size_t)CCtxParams->validateSequences; + + case ZSTD_c_splitAfterSequences: + BOUNDCHECK(ZSTD_c_splitAfterSequences, value); + CCtxParams->postBlockSplitter = (ZSTD_ParamSwitch_e)value; + return CCtxParams->postBlockSplitter; + + case ZSTD_c_blockSplitterLevel: + BOUNDCHECK(ZSTD_c_blockSplitterLevel, value); + CCtxParams->preBlockSplitter_level = value; + return (size_t)CCtxParams->preBlockSplitter_level; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_ParamSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return (size_t)CCtxParams->deterministicRefPrefix; + + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_ParamSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return (size_t)CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + assert(value>=0); + CCtxParams->maxBlockSize = (size_t)value; + return CCtxParams->maxBlockSize; + + case ZSTD_c_repcodeResolution: + BOUNDCHECK(ZSTD_c_repcodeResolution, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_ParamSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } @@ -15276,7 +20787,7 @@ size_t ZSTD_CCtxParams_getParameter( switch(param) { case ZSTD_c_format : - *value = CCtxParams->format; + *value = (int)CCtxParams->format; break; case ZSTD_c_compressionLevel : *value = CCtxParams->compressionLevel; @@ -15291,16 +20802,16 @@ size_t ZSTD_CCtxParams_getParameter( *value = (int)CCtxParams->cParams.chainLog; break; case ZSTD_c_searchLog : - *value = CCtxParams->cParams.searchLog; + *value = (int)CCtxParams->cParams.searchLog; break; case ZSTD_c_minMatch : - *value = CCtxParams->cParams.minMatch; + *value = (int)CCtxParams->cParams.minMatch; break; case ZSTD_c_targetLength : - *value = CCtxParams->cParams.targetLength; + *value = (int)CCtxParams->cParams.targetLength; break; case ZSTD_c_strategy : - *value = (unsigned)CCtxParams->cParams.strategy; + *value = (int)CCtxParams->cParams.strategy; break; case ZSTD_c_contentSizeFlag : *value = CCtxParams->fParams.contentSizeFlag; @@ -15315,10 +20826,10 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->forceWindow; break; case ZSTD_c_forceAttachDict : - *value = CCtxParams->attachDictPref; + *value = (int)CCtxParams->attachDictPref; break; case ZSTD_c_literalCompressionMode : - *value = CCtxParams->literalCompressionMode; + *value = (int)CCtxParams->literalCompressionMode; break; case ZSTD_c_nbWorkers : #ifndef ZSTD_MULTITHREAD @@ -15352,19 +20863,19 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->enableDedicatedDictSearch; break; case ZSTD_c_enableLongDistanceMatching : - *value = CCtxParams->ldmParams.enableLdm; + *value = (int)CCtxParams->ldmParams.enableLdm; break; case ZSTD_c_ldmHashLog : - *value = CCtxParams->ldmParams.hashLog; + *value = (int)CCtxParams->ldmParams.hashLog; break; case ZSTD_c_ldmMinMatch : - *value = CCtxParams->ldmParams.minMatchLength; + *value = (int)CCtxParams->ldmParams.minMatchLength; break; case ZSTD_c_ldmBucketSizeLog : - *value = CCtxParams->ldmParams.bucketSizeLog; + *value = (int)CCtxParams->ldmParams.bucketSizeLog; break; case ZSTD_c_ldmHashRateLog : - *value = CCtxParams->ldmParams.hashRateLog; + *value = (int)CCtxParams->ldmParams.hashRateLog; break; case ZSTD_c_targetCBlockSize : *value = (int)CCtxParams->targetCBlockSize; @@ -15384,6 +20895,30 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_validateSequences : *value = (int)CCtxParams->validateSequences; break; + case ZSTD_c_splitAfterSequences : + *value = (int)CCtxParams->postBlockSplitter; + break; + case ZSTD_c_blockSplitterLevel : + *value = CCtxParams->preBlockSplitter_level; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_repcodeResolution: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -15410,9 +20945,47 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( return 0; } -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) { - DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, (int)cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, (int)cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, (int)cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, (int)cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, (int)cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, (int)cparams.strategy), ""); + return 0; +} + +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} + +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} + +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, "Can't set pledgedSrcSize when not in init stage."); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; @@ -15428,9 +21001,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams( ZSTD_compressionParameters* cParams); /** - * Initializes the local dict using the requested parameters. - * NOTE: This does not use the pledged src size, because it may be used for more - * than one compression. + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. */ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) { @@ -15443,8 +21016,8 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) return 0; } if (dl->cdict != NULL) { - assert(cctx->cdict == dl->cdict); /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); return 0; } assert(dl->dictSize > 0); @@ -15464,40 +21037,44 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) } size_t ZSTD_CCtx_loadDictionary_advanced( - ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't load a dictionary when ctx is not in init stage."); DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); - ZSTD_clearAllDicts(cctx); /* in case one already exists */ - if (dict == NULL || dictSize == 0) /* no dictionary mode */ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ return 0; if (dictLoadMethod == ZSTD_dlm_byRef) { cctx->localDict.dict = dict; } else { + /* copy dictionary content inside CCtx to own its lifetime */ void* dictBuffer; RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "no malloc for static CCtx"); + "static CCtx can't allocate for an internal copy of dictionary"); dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); - RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); ZSTD_memcpy(dictBuffer, dict, dictSize); - cctx->localDict.dictBuffer = dictBuffer; - cctx->localDict.dict = dictBuffer; + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ } cctx->localDict.dictSize = dictSize; cctx->localDict.dictContentType = dictContentType; return 0; } -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( +size_t ZSTD_CCtx_loadDictionary_byReference( ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { return ZSTD_CCtx_loadDictionary_advanced( cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); } -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { return ZSTD_CCtx_loadDictionary_advanced( cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); @@ -15553,7 +21130,7 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't reset parameters only when not in init stage."); + "Reset parameters is only possible during init stage."); ZSTD_clearAllDicts(cctx); return ZSTD_CCtxParams_reset(&cctx->requestedParams); } @@ -15572,7 +21149,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); - BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + BOUNDCHECK(ZSTD_c_strategy, (int)cParams.strategy); return 0; } @@ -15582,11 +21159,12 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) { -# define CLAMP_TYPE(cParam, val, type) { \ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ - if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ - } +# define CLAMP_TYPE(cParam, val, type) \ + do { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } while (0) # define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) CLAMP(ZSTD_c_windowLog, cParams.windowLog); CLAMP(ZSTD_c_chainLog, cParams.chainLog); @@ -15644,19 +21222,62 @@ static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) * optimize `cPar` for a specified input (`srcSize` and `dictSize`). * mostly downsize to reduce memory consumption and initialization latency. * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. - * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_CParamMode_e`. * note : `srcSize==0` means 0! * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize, - ZSTD_cParamMode_e mode) + ZSTD_CParamMode_e mode, + ZSTD_ParamSwitch_e useRowMatchFinder) { const U64 minSrcSize = 513; /* (1<<9) + 1 */ const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); + /* Cascade the selected strategy down to the next-highest one built into + * this binary. */ +#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btultra2) { + cPar.strategy = ZSTD_btultra; + } + if (cPar.strategy == ZSTD_btultra) { + cPar.strategy = ZSTD_btopt; + } +#endif +#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btopt) { + cPar.strategy = ZSTD_btlazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btlazy2) { + cPar.strategy = ZSTD_lazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy2) { + cPar.strategy = ZSTD_lazy; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy) { + cPar.strategy = ZSTD_greedy; + } +#endif +#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_greedy) { + cPar.strategy = ZSTD_dfast; + } +#endif +#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_dfast) { + cPar.strategy = ZSTD_fast; + cPar.targetLength = 0; + } +#endif + switch (mode) { case ZSTD_cpm_unknown: case ZSTD_cpm_noAttachDict: @@ -15667,7 +21288,7 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, break; case ZSTD_cpm_createCDict: /* Assume a small source size when creating a dictionary - * with an unkown source size. + * with an unknown source size. */ if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) srcSize = minSrcSize; @@ -15685,8 +21306,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } /* resize windowLog if input is small enough, to use less memory */ - if ( (srcSize < maxWindowResize) - && (dictSize < maxWindowResize) ) { + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { U32 const tSize = (U32)(srcSize + dictSize); static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : @@ -15704,6 +21325,42 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } + } + + + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; + + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } + return cPar; } @@ -15714,11 +21371,11 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, { cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); } -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); static void ZSTD_overrideCParams( ZSTD_compressionParameters* cParams, @@ -15734,25 +21391,31 @@ static void ZSTD_overrideCParams( } ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { ZSTD_compressionParameters cParams; if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { - srcSizeHint = CCtxParams->srcSizeHint; + assert(CCtxParams->srcSizeHint>=0); + srcSizeHint = (U64)CCtxParams->srcSizeHint; } cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); - if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); assert(!ZSTD_checkCParams(cParams)); /* srcSizeHint == 0 means 0 */ - return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); } static size_t ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_ParamSwitch_e useRowMatchFinder, + const int enableDedicatedDictSearch, const U32 forCCtx) { - size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; @@ -15762,43 +21425,61 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + hSize * sizeof(U32) + h3Size * sizeof(U32); size_t const optPotentialSpace = - ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32)) - + ZSTD_cwksp_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned64_alloc_size(hSize) + : 0; size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", (U32)chainSize, (U32)hSize, (U32)h3Size); - return tableSpace + optSpace; + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} + +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; } static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_compressionParameters* cParams, const ldmParams_t* ldmParams, const int isStatic, + const ZSTD_ParamSwitch_e useRowMatchFinder, const size_t buffInSize, const size_t buffOutSize, - const U64 pledgedSrcSize) + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) { - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cParams->windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (cParams->minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) - + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(SeqDef)) + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); - size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); - size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, /* forCCtx */ 1); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); - size_t const ldmSeqSpace = ldmParams->enableLdm ? - ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) @@ -15806,15 +21487,21 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; + size_t const neededSpace = cctxSpace + - entropySpace + + tmpWorkSpace + blockStateSpace + ldmSpace + ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + bufferSpace + + externalSeqSpace; DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); return neededSpace; @@ -15824,25 +21511,45 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); /* estimateCCtxSize is for one-shot compression. So no buffers should * be needed. However, we still allocate two 0-sized buffers, which can * take space under ASAN. */ return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); } size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) { - ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } } static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - return ZSTD_estimateCCtxSize_usingCParams(cParams); + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; } size_t ZSTD_estimateCCtxSize(int compressionLevel) @@ -15850,6 +21557,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel) int level; size_t memBudget = 0; for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ size_t const newMB = ZSTD_estimateCCtxSize_internal(level); if (newMB > memBudget) memBudget = newMB; } @@ -15861,24 +21569,36 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) ? ((size_t)1 << cParams.windowLog) + blockSize : 0; size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, inBuffSize, outBuffSize, - ZSTD_CONTENTSIZE_UNKNOWN); + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); } } size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) { - ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } } static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) @@ -15966,7 +21686,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) * Invalidate all the matches in the match finder tables. * Requires nextSrc and base to be set (can be NULL). */ -static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +static void ZSTD_invalidateMatchState(ZSTD_MatchState_t* ms) { ZSTD_window_clear(&ms->window); @@ -16003,26 +21723,47 @@ typedef enum { ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); +} + +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); +} + static size_t -ZSTD_reset_matchState(ZSTD_matchState_t* ms, +ZSTD_reset_matchState(ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, + const ZSTD_ParamSwitch_e useRowMatchFinder, const ZSTD_compResetPolicy_e crp, const ZSTD_indexResetPolicy_e forceResetIndex, const ZSTD_resetTarget_e forWho) { - size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); if (forceResetIndex == ZSTDirp_reset) { ZSTD_window_init(&ms->window); ZSTD_cwksp_mark_tables_dirty(ws); } ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; ZSTD_invalidateMatchState(ms); @@ -16044,22 +21785,42 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_cwksp_clean_tables(ws); } + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); } ms->cParams = *cParams; RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); - return 0; } @@ -16076,61 +21837,86 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) +{ + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; +} + /*! ZSTD_resetCCtx_internal() : - note : `params` are assumed fully validated at this stage */ + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. + */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, - ZSTD_CCtx_params params, + ZSTD_CCtx_params const* params, U64 const pledgedSrcSize, + size_t const loadedDictSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { ZSTD_cwksp* const ws = &zc->workspace; - DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", - (U32)pledgedSrcSize, params.cParams.windowLog); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->postBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); zc->isFirstBlock = 1; - if (params.ldmParams.enableLdm) { + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->postBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* Adjust long distance matching parameters */ - ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); - assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashRateLog < 32); + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); } - { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(params)); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; - size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) ? windowSize + blockSize : 0; - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); ZSTD_indexResetPolicy_e needsIndexReset = - (!indexTooClose && zc->initialized) ? ZSTDirp_continue : ZSTDirp_reset; + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; size_t const neededSpace = ZSTD_estimateCCtxSize_usingCCtxParams_internal( - ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, - buffInSize, buffOutSize, pledgedSrcSize); + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); - /* Check if workspace is large enough, alloc a new one if needed */ - { + { /* Check if workspace is large enough, alloc a new one if needed */ int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); - + int resizeWorkspace = workspaceTooSmall || workspaceWasteful; DEBUGLOG(4, "Need %zu B workspace", neededSpace); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); - if (workspaceTooSmall || workspaceWasteful) { + if (resizeWorkspace) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", ZSTD_cwksp_sizeof(ws) >> 10, neededSpace >> 10); @@ -16144,22 +21930,23 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(5, "reserving object space"); /* Statically sized space. - * entropyWorkspace never moves, + * tmpWorkspace never moves, * though prev/next block swap places */ assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); - RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace"); + zc->tmpWkspSize = TMP_WORKSPACE_SIZE; } } ZSTD_cwksp_clear(ws); /* init params */ - zc->appliedParams = params; - zc->blockState.matchState.cParams = params.cParams; + zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; zc->consumedSrcSize = 0; zc->producedCSize = 0; @@ -16167,7 +21954,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->appliedParams.fParams.contentSizeFlag = 0; DEBUGLOG(4, "pledged content size : %u ; flag : %u", (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); - zc->blockSize = blockSize; + zc->blockSizeMax = blockSize; XXH64_reset(&zc->xxhState, 0); zc->stage = ZSTDcs_init; @@ -16176,13 +21963,46 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + zc->seqStore.sequencesStart = (SeqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(SeqDef)); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* reserve space for block-level external sequences */ + if (ZSTD_hasExtSeqProd(params)) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->extSeqBufCapacity = maxNbExternalSeq; + zc->extSeqBuf = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + + /* buffers */ + /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; - /* buffers */ zc->bufferedPolicy = zbuff; zc->inBuffSize = buffInSize; zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); @@ -16190,11 +22010,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* TODO: avoid memset? */ size_t const numBuckets = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); } @@ -16205,37 +22025,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); - - FORWARD_IF_ERROR(ZSTD_reset_matchState( - &zc->blockState.matchState, - ws, - ¶ms.cParams, - crp, - needsIndexReset, - ZSTD_resetTarget_CCtx), ""); - - /* ldm hash table */ - if (params.ldmParams.enableLdm) { - /* TODO: avoid memset? */ - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); - ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); - zc->maxNbLdmSequences = maxNbLdmSeq; - - ZSTD_window_init(&zc->ldmState.window); - ZSTD_window_clear(&zc->ldmState.window); - zc->ldmState.loadedDictEnd = 0; - } - - /* Due to alignment, when reusing a workspace, we can actually consume - * up to 3 extra bytes for alignment. See the comments in zstd_cwksp.h - */ - assert(ZSTD_cwksp_used(ws) >= neededSpace && - ZSTD_cwksp_used(ws) <= neededSpace + 3); DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); + zc->initialized = 1; return 0; @@ -16291,6 +22084,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; @@ -16304,9 +22099,12 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, } params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, - cdict->dictContentSize, ZSTD_cpm_attachDict); + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); params.cParams.windowLog = windowLog; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, ZSTDcrp_makeClean, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); } @@ -16341,6 +22139,22 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, return 0; } +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); + } +} + static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, @@ -16350,15 +22164,17 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; assert(!cdict->matchState.dedicatedDictSearch); - - DEBUGLOG(4, "copying dictionary into context"); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); /* Copy only compression parameters related to tables. */ params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); @@ -16366,21 +22182,37 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, } ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); /* copy tables */ - { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - ZSTD_memcpy(cctx->blockState.matchState.hashTable, - cdict->matchState.hashTable, - hSize * sizeof(U32)); - ZSTD_memcpy(cctx->blockState.matchState.chainTable, - cdict->matchState.chainTable, - chainSize * sizeof(U32)); + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize; + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; + } } /* Zero the hashTable3, since the cdict never fills it */ - { int const h3log = cctx->blockState.matchState.hashLog3; + assert(cctx->blockState.matchState.hashLog3 <= 31); + { U32 const h3log = cctx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); @@ -16389,8 +22221,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, ZSTD_cwksp_mark_tables_clean(&cctx->workspace); /* copy dictionary offsets */ - { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; - ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + { ZSTD_MatchState_t const* srcMatchState = &cdict->matchState; + ZSTD_MatchState_t* dstMatchState = &cctx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; @@ -16440,16 +22272,23 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(5, "ZSTD_copyCCtx_internal"); RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, "Can't copy a ctx that's not in init stage."); - + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); { ZSTD_CCtx_params params = dstCCtx->requestedParams; /* Copy only compression parameters related to tables. */ params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.postBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.postBlockSplitter = srcCCtx->appliedParams.postBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; params.fParams = fParams; - ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); @@ -16461,9 +22300,13 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); /* copy tables */ - { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - int const h3log = srcCCtx->blockState.matchState.hashLog3; + U32 const h3log = srcCCtx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, @@ -16481,8 +22324,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, /* copy dictionary offsets */ { - const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; - ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + const ZSTD_MatchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_MatchState_t* dstMatchState = &dstCCtx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; @@ -16528,11 +22371,13 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa int const nbRows = (int)size / ZSTD_ROWSIZE; int cellNb = 0; int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ - assert(size < (1U<<31)); /* can be casted to int */ + assert(size < (1U<<31)); /* can be cast to int */ #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) - /* To validate that the table re-use logic is sound, and that we don't + /* To validate that the table reuse logic is sound, and that we don't * access table space that we haven't cleaned, we re-"poison" the table * space every time we mark it dirty. * @@ -16547,12 +22392,17 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa for (rowNb=0 ; rowNb < nbRows ; rowNb++) { int column; for (column=0; columncParams.hashLog; ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); } - if (params->cParams.strategy != ZSTD_fast) { + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { U32 const chainSize = (U32)1 << params->cParams.chainLog; if (params->cParams.strategy == ZSTD_btlazy2) ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); @@ -16596,26 +22446,32 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par /* See doc/zstd_compression_format.md for detailed format description */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr) { - const seqDef* const sequences = seqStorePtr->sequencesStart; + const SeqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; BYTE* const ofCodeTable = seqStorePtr->ofCode; BYTE* const mlCodeTable = seqStorePtr->mlCode; U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); U32 u; + int longOffsets = 0; assert(nbSeq <= seqStorePtr->maxNbSeq); for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; } - if (seqStorePtr->longLengthID==1) + if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; - if (seqStorePtr->longLengthID==2) + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; } /* ZSTD_useTargetCBlockSize(): @@ -16628,53 +22484,209 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) return (cctxParams->targetCBlockSize != 0); } -/* ZSTD_entropyCompressSequences_internal(): - * actually compresses both literals and sequences */ -MEM_STATIC size_t -ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - void* entropyWorkspace, size_t entropyWkspSize, - const int bmi2) +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (postBlockSplitter=%d)", cctxParams->postBlockSplitter); + assert(cctxParams->postBlockSplitter != ZSTD_ps_auto); + return (cctxParams->postBlockSplitter == ZSTD_ps_enable); +} + +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics( + const SeqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; + + stats.lastCountSize = 0; + /* convert length/distances into codes */ + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} + +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal( + void* dst, size_t dstCapacity, + const void* literals, size_t litSize, + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) { - const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; ZSTD_strategy const strategy = cctxParams->cParams.strategy; unsigned* count = (unsigned*)entropyWorkspace; FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; - U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ - const seqDef* const sequences = seqStorePtr->sequencesStart; + const SeqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); const BYTE* const ofCodeTable = seqStorePtr->ofCode; const BYTE* const llCodeTable = seqStorePtr->llCode; const BYTE* const mlCodeTable = seqStorePtr->mlCode; BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - BYTE* seqHead; - BYTE* lastNCount = NULL; + size_t lastCountSize; + int longOffsets = 0; entropyWorkspace = count + (MaxSeq + 1); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); - DEBUGLOG(4, "ZSTD_entropyCompressSequences_internal (nbSeq=%zu)", nbSeq); + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); /* Compress literals */ - { const BYTE* const literals = seqStorePtr->litStart; - size_t const litSize = (size_t)(seqStorePtr->lit - literals); + { size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + size_t const cSize = ZSTD_compressLiterals( - &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, - ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, entropyWorkspace, entropyWkspSize, - bmi2); + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); assert(cSize <= dstCapacity); op += cSize; @@ -16700,95 +22712,20 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); return (size_t)(op - ostart); } - - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - assert(op <= oend); - - /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); - /* build CTable for Literal Lengths */ - { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building LL table"); - nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, - count, max, mostFrequent, nbSeq, - LLFSELog, prevEntropy->fse.litlengthCTable, - LL_defaultNorm, LL_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(set_basic < set_compressed && set_rle < set_compressed); - assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, - sizeof(prevEntropy->fse.litlengthCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); - if (LLtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); - } } - /* build CTable for Offsets */ - { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp( - count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; - DEBUGLOG(5, "Building OF table"); - nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, - count, max, mostFrequent, nbSeq, - OffFSELog, prevEntropy->fse.offcodeCTable, - OF_defaultNorm, OF_defaultNormLog, - defaultPolicy, strategy); - assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, - sizeof(prevEntropy->fse.offcodeCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); - if (Offtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); - } } - /* build CTable for MatchLengths */ - { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp( - count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); - nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, - count, max, mostFrequent, nbSeq, - MLFSELog, prevEntropy->fse.matchlengthCTable, - ML_defaultNorm, ML_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, - ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, - sizeof(prevEntropy->fse.matchlengthCTable), - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); - if (MLtype == set_compressed) - lastNCount = op; - op += countSize; - assert(op <= oend); - } } - - *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + { BYTE* const seqHead = op++; + /* build stats for sequences */ + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + longOffsets = stats.longOffsets; + } { size_t const bitstreamSize = ZSTD_encodeSequences( op, (size_t)(oend - op), @@ -16808,9 +22745,9 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, * In this exceedingly rare case, we will simply emit an uncompressed * block, since it isn't worth optimizing. */ - if (lastNCount && (op - lastNCount) < 4) { - /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(op - lastNCount == 3); + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " "emitting an uncompressed block."); return 0; @@ -16821,116 +22758,275 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, return (size_t)(op - ostart); } -MEM_STATIC size_t -ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - size_t srcSize, - void* entropyWorkspace, size_t entropyWkspSize, - int bmi2) +static size_t +ZSTD_entropyCompressSeqStore_wExtLitBuffer( + void* dst, size_t dstCapacity, + const void* literals, size_t litSize, + size_t blockSize, + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) { - size_t const cSize = ZSTD_entropyCompressSequences_internal( - seqStorePtr, prevEntropy, nextEntropy, cctxParams, + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( dst, dstCapacity, + literals, litSize, + seqStorePtr, prevEntropy, nextEntropy, cctxParams, entropyWorkspace, entropyWkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. */ - if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) + if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); return 0; /* block not compressed */ - FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSequences_internal failed"); + } + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); /* Check compressibility */ - { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + { size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy); if (cSize >= maxCSize) return 0; /* block not compressed */ } - DEBUGLOG(4, "ZSTD_entropyCompressSequences() cSize: %zu\n", cSize); + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); return cSize; } +static size_t +ZSTD_entropyCompressSeqStore( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + return ZSTD_entropyCompressSeqStore_wExtLitBuffer( + dst, dstCapacity, + seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart), + srcSize, + seqStorePtr, + prevEntropy, nextEntropy, + cctxParams, + entropyWorkspace, entropyWkspSize, + bmi2); +} + /* ZSTD_selectBlockCompressor() : * Not static, but internal use only (used by long distance matcher) * assumption : strat is a valid strategy */ -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) +ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) { - static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { { ZSTD_compressBlock_fast /* default for 0 */, ZSTD_compressBlock_fast, - ZSTD_compressBlock_doubleFast, - ZSTD_compressBlock_greedy, - ZSTD_compressBlock_lazy, - ZSTD_compressBlock_lazy2, - ZSTD_compressBlock_btlazy2, - ZSTD_compressBlock_btopt, - ZSTD_compressBlock_btultra, - ZSTD_compressBlock_btultra2 }, + ZSTD_COMPRESSBLOCK_DOUBLEFAST, + ZSTD_COMPRESSBLOCK_GREEDY, + ZSTD_COMPRESSBLOCK_LAZY, + ZSTD_COMPRESSBLOCK_LAZY2, + ZSTD_COMPRESSBLOCK_BTLAZY2, + ZSTD_COMPRESSBLOCK_BTOPT, + ZSTD_COMPRESSBLOCK_BTULTRA, + ZSTD_COMPRESSBLOCK_BTULTRA2 + }, { ZSTD_compressBlock_fast_extDict /* default for 0 */, ZSTD_compressBlock_fast_extDict, - ZSTD_compressBlock_doubleFast_extDict, - ZSTD_compressBlock_greedy_extDict, - ZSTD_compressBlock_lazy_extDict, - ZSTD_compressBlock_lazy2_extDict, - ZSTD_compressBlock_btlazy2_extDict, - ZSTD_compressBlock_btopt_extDict, - ZSTD_compressBlock_btultra_extDict, - ZSTD_compressBlock_btultra_extDict }, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT, + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT + }, { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, ZSTD_compressBlock_fast_dictMatchState, - ZSTD_compressBlock_doubleFast_dictMatchState, - ZSTD_compressBlock_greedy_dictMatchState, - ZSTD_compressBlock_lazy_dictMatchState, - ZSTD_compressBlock_lazy2_dictMatchState, - ZSTD_compressBlock_btlazy2_dictMatchState, - ZSTD_compressBlock_btopt_dictMatchState, - ZSTD_compressBlock_btultra_dictMatchState, - ZSTD_compressBlock_btultra_dictMatchState }, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE + }, { NULL /* default for 0 */, NULL, NULL, - ZSTD_compressBlock_greedy_dedicatedDictSearch, - ZSTD_compressBlock_lazy_dedicatedDictSearch, - ZSTD_compressBlock_lazy2_dedicatedDictSearch, + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH, NULL, NULL, NULL, NULL } }; - ZSTD_blockCompressor selectedCompressor; + ZSTD_BlockCompressor_f selectedCompressor; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); - selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + DEBUGLOG(5, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_BlockCompressor_f rowBasedBlockCompressors[4][3] = { + { + ZSTD_COMPRESSBLOCK_GREEDY_ROW, + ZSTD_COMPRESSBLOCK_LAZY_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW + } + }; + DEBUGLOG(5, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + } assert(selectedCompressor != NULL); return selectedCompressor; } -static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, +static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr, const BYTE* anchor, size_t lastLLSize) { ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } -void ZSTD_resetSeqStore(seqStore_t* ssPtr) +void ZSTD_resetSeqStore(SeqStore_t* ssPtr) { ssPtr->lit = ssPtr->litStart; ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthID = 0; + ssPtr->longLengthType = ZSTD_llt_none; } -typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); + + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); + + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; + } + + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; + + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } + + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); + + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} + +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; +} + +/** + * Function to validate sequences produced by a block compressor. + */ +static void ZSTD_validateSeqStore(const SeqStore_t* seqStore, const ZSTD_compressionParameters* cParams) +{ +#if DEBUGLEVEL >= 1 + const SeqDef* seq = seqStore->sequencesStart; + const SeqDef* const seqEnd = seqStore->sequences; + size_t const matchLenLowerBound = cParams->minMatch == 3 ? 3 : 4; + for (; seq < seqEnd; ++seq) { + const ZSTD_SequenceLength seqLength = ZSTD_getSequenceLength(seqStore, seq); + assert(seqLength.matchLength >= matchLenLowerBound); + (void)seqLength; + (void)matchLenLowerBound; + } +#else + (void)seqStore; + (void)cParams; +#endif +} + +static size_t +ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch); + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_BuildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) { - ZSTD_matchState_t* const ms = &zc->blockState.matchState; + ZSTD_MatchState_t* const ms = &zc->blockState.matchState; DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); } else { @@ -16965,16 +23061,34 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; } if (zc->externSeqStore.pos < zc->externSeqStore.size) { - assert(!zc->appliedParams.ldmParams.enableLdm); + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + /* Updates ldmSeqStore.pos */ lastLLSize = ZSTD_ldm_blockCompress(&zc->externSeqStore, ms, &zc->seqStore, zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, src, srcSize); assert(zc->externSeqStore.pos <= zc->externSeqStore.size); - } else if (zc->appliedParams.ldmParams.enableLdm) { - rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + RawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; @@ -16987,90 +23101,196 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) ZSTD_ldm_blockCompress(&ldmSeqStore, ms, &zc->seqStore, zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, src, srcSize); assert(ldmSeqStore.pos == ldmSeqStore.size); - } else { /* not long range mode */ - ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode); + } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { + assert( + zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->appliedParams.extSeqProdFunc != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( + zc->appliedParams.extSeqProdState, + zc->extSeqBuf, + zc->extSeqBufCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->extSeqBuf, + nbExternalSeqs, + zc->extSeqBufCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_SequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_transferSequences_wBlockDelim( + zc, &seqPos, + zc->extSeqBuf, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } + + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } + + /* Fallback to software matchfinder */ + { ZSTD_BlockCompressor_f const blockCompressor = + ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ + ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); ms->ldmSeqStore = NULL; lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); } { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); } } + ZSTD_validateSeqStore(&zc->seqStore, &zc->appliedParams.cParams); return ZSTDbss_compress; } -static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) { - const seqStore_t* seqStore = ZSTD_getSeqStore(zc); - const seqDef* seqStoreSeqs = seqStore->sequencesStart; - size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; - size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); - size_t literalsRead = 0; - size_t lastLLSize; + const SeqDef* inSeqs = seqStore->sequencesStart; + const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs); + const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); - ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; + const size_t nbOutSequences = nbInSequences + 1; + size_t nbOutLiterals = 0; + Repcodes_t repcodes; size_t i; - repcodes_t updatedRepcodes; - assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); - /* Ensure we have enough space for last literals "sequence" */ - assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); - ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); - for (i = 0; i < seqStoreSeqSize; ++i) { - U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM; - outSeqs[i].litLength = seqStoreSeqs[i].litLength; - outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; + /* Bounds check that we have enough space for every input sequence + * and the block delimiter + */ + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + RETURN_ERROR_IF( + nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), + dstSize_tooSmall, + "Not enough space to copy sequences"); + + ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes)); + for (i = 0; i < nbInSequences; ++i) { + U32 rawOffset; + outSeqs[i].litLength = inSeqs[i].litLength; + outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH; outSeqs[i].rep = 0; + /* Handle the possible single length >= 64K + * There can only be one because we add MINMATCH to every match length, + * and blocks are at most 128K. + */ if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { outSeqs[i].matchLength += 0x10000; } } - if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { - /* Derive the correct offset corresponding to a repcode */ - outSeqs[i].rep = seqStoreSeqs[i].offset; + /* Determine the raw offset given the offBase, which may be a repcode. */ + if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) { + const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase); + assert(repcode > 0); + outSeqs[i].rep = repcode; if (outSeqs[i].litLength != 0) { - rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + rawOffset = repcodes.rep[repcode - 1]; } else { - if (outSeqs[i].rep == 3) { - rawOffset = updatedRepcodes.rep[0] - 1; + if (repcode == 3) { + assert(repcodes.rep[0] > 1); + rawOffset = repcodes.rep[0] - 1; } else { - rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; + rawOffset = repcodes.rep[repcode]; } } + } else { + rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase); } outSeqs[i].offset = rawOffset; - /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode - so we provide seqStoreSeqs[i].offset - 1 */ - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, - seqStoreSeqs[i].offset - 1, - seqStoreSeqs[i].litLength == 0); - literalsRead += outSeqs[i].litLength; + + /* Update repcode history for the sequence */ + ZSTD_updateRep(repcodes.rep, + inSeqs[i].offBase, + inSeqs[i].litLength == 0); + + nbOutLiterals += outSeqs[i].litLength; } /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker * for the block boundary, according to the API. */ - assert(seqStoreLiteralsSize >= literalsRead); - lastLLSize = seqStoreLiteralsSize - literalsRead; - outSeqs[i].litLength = (U32)lastLLSize; - outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; - seqStoreSeqSize++; - zc->seqCollector.seqIndex += seqStoreSeqSize; + assert(nbInLiterals >= nbOutLiterals); + { + const size_t lastLLSize = nbInLiterals - nbOutLiterals; + outSeqs[nbInSequences].litLength = (U32)lastLLSize; + outSeqs[nbInSequences].matchLength = 0; + outSeqs[nbInSequences].offset = 0; + assert(nbOutSequences == nbInSequences + 1); + } + seqCollector->seqIndex += nbOutSequences; + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + + return 0; +} + +size_t ZSTD_sequenceBound(size_t srcSize) { + const size_t maxNbSeq = (srcSize / ZSTD_MINMATCH_MIN) + 1; + const size_t maxNbDelims = (srcSize / ZSTD_BLOCKSIZE_MAX_MIN) + 1; + return maxNbSeq + maxNbDelims; } size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { const size_t dstCapacity = ZSTD_compressBound(srcSize); - void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + void* dst; /* Make C90 happy. */ SeqCollector seqCollector; + { + int targetCBlockSize; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetCBlockSize, &targetCBlockSize), ""); + RETURN_ERROR_IF(targetCBlockSize != 0, parameter_unsupported, "targetCBlockSize != 0"); + } + { + int nbWorkers; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers), ""); + RETURN_ERROR_IF(nbWorkers != 0, parameter_unsupported, "nbWorkers != 0"); + } + dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); seqCollector.collectSequences = 1; @@ -17079,8 +23299,12 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, seqCollector.maxSequences = outSeqsSize; zc->seqCollector = seqCollector; - ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); - ZSTD_customFree(dst, ZSTD_defaultCMem); + { + const size_t ret = ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + FORWARD_IF_ERROR(ret, "ZSTD_compress2 failed"); + } + assert(zc->seqCollector.seqIndex <= ZSTD_sequenceBound(srcSize)); return zc->seqCollector.seqIndex; } @@ -17109,19 +23333,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) { const size_t unrollMask = unrollSize - 1; const size_t prefixLength = length & unrollMask; size_t i; - size_t u; if (length == 1) return 1; /* Check if prefix is RLE first before using unrolled loop */ if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { return 0; } for (i = prefixLength; i != length; i += unrollSize) { + size_t u; for (u = 0; u < unrollSize; u += sizeof(size_t)) { if (MEM_readST(ip + i + u) != valueST) { return 0; - } - } - } + } } } return 1; } @@ -17129,7 +23351,7 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) { * This is just a heuristic based on the compressibility. * It may return both false positives and false negatives. */ -static int ZSTD_maybeRLE(seqStore_t const* seqStore) +static int ZSTD_maybeRLE(SeqStore_t const* seqStore) { size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); @@ -17137,20 +23359,790 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore) return nbSeqs < 4 && nbLits < 10; } -static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc) +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) { - ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; - zc->blockState.prevCBlock = zc->blockState.nextCBlock; - zc->blockState.nextCBlock = tmp; + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, U32 frame) +/* Writes the block header */ +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) { - /* This the upper bound for the length of an rle block. - * This isn't the actual upper bound. Finding the real threshold - * needs further investigation. + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); +} + +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = LitHufLog; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Scan input and build symbol stats */ + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + /* only one literal symbol */ + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } +} + + +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; +} + +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const SeqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * @return : 0 on success, or an error code + * Note : also employed in superblock + */ +size_t ZSTD_buildBlockEntropyStats( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; + + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize, hufFlags); + + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; +} + +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t +ZSTD_estimateBlockSize_symbolType(SymbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; + } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} + +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * @return: estimated compressed size of the seqStore, or a zstd error. + */ +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->tmpWorkspace, zc->tmpWkspSize), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->tmpWorkspace, zc->tmpWkspSize, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore) +{ + size_t literalsBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + SeqDef const seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; + } } + return literalsBytes; +} + +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore) +{ + size_t matchBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + SeqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } } + return matchBytes; +} + +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore, + const SeqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) +{ + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; + } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + assert(resultSeqStore->lit == originalSeqStore->lit); + } else { + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} + +/** + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) +{ + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ + return rep[0] - 1; + } + return rep[adjustedRepCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void +ZSTD_seqStore_resolveOffCodes(Repcodes_t* const dRepcodes, Repcodes_t* const cRepcodes, + const SeqStore_t* const seqStore, U32 const nbSeq) +{ + U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; + for (; idx < nbSeq; ++idx) { + SeqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); + } + } + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); + } +} + +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const SeqStore_t* const seqStore, + Repcodes_t* const dRep, Repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) +{ + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + Repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + /* Sequence collection not supported when block splitting */ + if (zc->seqCollector.collectSequences) { + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, seqStore, dRepOriginal.rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cSize); + } + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const SeqStore_t* origSeqStore) +{ + SeqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + SeqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + SeqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); + return; + } + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); + } +} + +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. + * + * @return: number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; + if (nbSeq <= 4) { + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); + /* Refuse to try and split anything with less than 4 sequences */ + return 0; + } + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; +} + +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. + */ +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) +{ + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + Repcodes_t dRep; + Repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(SeqStore_t)); + + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t)); + return cSize; +} + +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) +{ + U32 nbSeq; + size_t cSize; + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.postBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; +} + +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. */ const U32 rleMaxLength = 25; size_t cSize; @@ -17162,30 +24154,28 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); - if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + if (bss == ZSTDbss_noCompress) { + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = 0; + goto out; + } } if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - ZSTD_confirmRepcodesAndEntropyTables(zc); + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.prevCBlock->rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); return 0; } /* encode sequences and literals */ - cSize = ZSTD_entropyCompressSequences(&zc->seqStore, + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, dst, dstCapacity, srcSize, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, zc->bmi2); - if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - return 0; - } - - if (frame && /* We don't want to emit our first block as a RLE even if it qualifies because * doing so will cause the decoder (cli only) to throw a "should consume all input error." @@ -17201,7 +24191,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, out: if (!ZSTD_isError(cSize) && cSize > 1) { - ZSTD_confirmRepcodesAndEntropyTables(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); } /* We check that dictionaries have offset codes available for the first * block. After the first block, the offcode table might not have large @@ -17248,18 +24238,19 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, * * cSize >= blockBound(srcSize): We have expanded the block too much so * emit an uncompressed block. */ - { - size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); if (cSize != ERROR(dstSize_tooSmall)) { - size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { - ZSTD_confirmRepcodesAndEntropyTables(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); return cSize; } } } - } + } /* if (bss == ZSTDbss_compress)*/ DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); /* Superblock compression failed, attempt to emit a single no compress block. @@ -17288,15 +24279,15 @@ static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, return cSize; } -static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, +static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, void const* ip, void const* iend) { - if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { - U32 const maxDist = (U32)1 << params->cParams.windowLog; - U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); @@ -17312,43 +24303,87 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, } } +/**** skipping file: zstd_preSplit.h ****/ + +static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, ZSTD_strategy strat, S64 savings) +{ + /* split level based on compression strategy, from `fast` to `btultra2` */ + static const int splitLevels[] = { 0, 0, 1, 2, 2, 3, 3, 4, 4, 4 }; + /* note: conservatively only split full blocks (128 KB) currently. + * While it's possible to go lower, let's keep it simple for a first implementation. + * Besides, benefits of splitting are reduced when blocks are already small. + */ + if (srcSize < 128 KB || blockSizeMax < 128 KB) + return MIN(srcSize, blockSizeMax); + /* do not split incompressible data though: + * require verified savings to allow pre-splitting. + * Note: as a consequence, the first full block is not split. + */ + if (savings < 3) { + DEBUGLOG(6, "don't attempt splitting: savings (%i) too low", (int)savings); + return 128 KB; + } + /* apply @splitLevel, or use default value (which depends on @strat). + * note that splitting heuristic is still conditioned by @savings >= 3, + * so the first block will not reach this code path */ + if (splitLevel == 1) return 128 KB; + if (splitLevel == 0) { + assert(ZSTD_fast <= strat && strat <= ZSTD_btultra2); + splitLevel = splitLevels[strat]; + } else { + assert(2 <= splitLevel && splitLevel <= 6); + splitLevel -= 2; + } + return ZSTD_splitBlock(src, blockSizeMax, splitLevel, cctx->tmpWorkspace, cctx->tmpWkspSize); +} + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. * Frame is supposed already started (header already produced) -* @return : compressed size, or an error code +* @return : compressed size, or an error code */ -static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastFrameChunk) { - size_t blockSize = cctx->blockSize; + size_t blockSizeMax = cctx->blockSizeMax; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize; assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + DEBUGLOG(5, "ZSTD_compress_frameChunk (srcSize=%u, blockSizeMax=%u)", (unsigned)srcSize, (unsigned)blockSizeMax); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { - ZSTD_matchState_t* const ms = &cctx->blockState.matchState; - U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; + size_t const blockSize = ZSTD_optimalBlockSize(cctx, + ip, remaining, + blockSizeMax, + cctx->appliedParams.preBlockSplitter_level, + cctx->appliedParams.cParams.strategy, + savings); + U32 const lastBlock = lastFrameChunk & (blockSize == remaining); + assert(blockSize <= remaining); - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, dstSize_tooSmall, "not enough space to store compressed block"); - if (remaining < blockSize) blockSize = remaining; ZSTD_overflowCorrectIfNeeded( ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; @@ -17359,6 +24394,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); assert(cSize > 0); assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); } else { cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, @@ -17375,8 +24414,23 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } - } + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + /* @savings is employed to ensure that splitting doesn't worsen expansion of incompressible data. + * Without splitting, the maximum expansion is 3 bytes per full block. + * An adversarial input could attempt to fudge the split detector, + * and make it split incompressible data, resulting in more block headers. + * Note that, since ZSTD_COMPRESSBOUND() assumes a worst case scenario of 1KB per block, + * and the splitter never creates blocks that small (current lower limit is 8 KB), + * there is already no risk to expand beyond ZSTD_COMPRESSBOUND() limit. + * But if the goal is to not expand by more than 3-bytes per 128 KB full block, + * then yes, it becomes possible to make the block splitter oversplit incompressible data. + * Using @savings, we enforce an even more conservative condition, + * requiring the presence of enough savings (at least 3 bytes) to authorize splitting, + * otherwise only full blocks are used. + * But being conservative is fine, + * since splitting barely compressible blocks is not fruitful anyway */ + savings += (S64)blockSize - (S64)cSize; ip += blockSize; assert(remaining >= blockSize); @@ -17395,8 +24449,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) -{ BYTE* const op = (BYTE*)dst; + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, U32 dictID) +{ + BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ U32 const checksumFlag = params->fParams.checksumFlag>0; @@ -17421,7 +24477,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, if (!singleSegment) op[pos++] = windowLogByte; switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : op[pos] = (BYTE)(dictID); pos++; break; case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; @@ -17429,7 +24487,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, } switch(fcsCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; @@ -17473,19 +24533,15 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) } } -size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) +void ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) { - RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong, - "wrong cctx stage"); - RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, - parameter_unsupported, - "incompatible with ldm"); + assert(cctx->stage == ZSTDcs_init); + assert(nbSeq == 0 || cctx->appliedParams.ldmParams.enableLdm != ZSTD_ps_enable); cctx->externSeqStore.seq = seq; cctx->externSeqStore.size = nbSeq; cctx->externSeqStore.capacity = nbSeq; cctx->externSeqStore.pos = 0; cctx->externSeqStore.posInSequence = 0; - return 0; } @@ -17494,7 +24550,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, const void* src, size_t srcSize, U32 frame, U32 lastFrameChunk) { - ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; size_t fhSize = 0; DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", @@ -17514,11 +24570,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!srcSize) return fhSize; /* do not generate an empty block if no input */ - if (!ZSTD_window_update(&ms->window, src, srcSize)) { + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; ms->nextToUpdate = ms->window.dictLimit; } - if (cctx->appliedParams.ldmParams.enableLdm) { - ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); } if (!frame) { @@ -17528,7 +24585,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, src, (BYTE const*)src + srcSize); } - DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSizeMax); { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); @@ -17549,100 +24606,189 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, } } -size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} -size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) { ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; assert(!ZSTD_checkCParams(cParams)); - return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); } -size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); - { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + /*! ZSTD_loadDictionaryContent() : * @return : 0, or an error code */ -static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, - ldmState_t* ls, - ZSTD_cwksp* ws, - ZSTD_CCtx_params const* params, - const void* src, size_t srcSize, - ZSTD_dictTableLoadMethod_e dtlm) +static size_t +ZSTD_loadDictionaryContent(ZSTD_MatchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; - ZSTD_window_update(&ms->window, src, srcSize); - ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + /* Assert that the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); - if (params->ldmParams.enableLdm && ls != NULL) { - ZSTD_window_update(&ls->window, src, srcSize); - ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + { /* Ensure large dictionaries can't cause index overflow */ + + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } } - /* Assert that we the ms params match the params we're being given */ - ZSTD_assertEqualCParams(params->cParams, ms->cParams); + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); + + DEBUGLOG(4, "ZSTD_loadDictionaryContent: useRowMatchFinder=%d", (int)params->useRowMatchFinder); + + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ + DEBUGLOG(4, "ZSTD_loadDictionaryContent: Trigger loadLdmDict"); + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + DEBUGLOG(4, "ZSTD_loadDictionaryContent: ZSTD_ldm_fillHashTable completes"); + } + + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + { U32 maxDictSize = 1U << MIN(MAX(params->cParams.hashLog + 3, params->cParams.chainLog + 1), 31); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; if (srcSize <= HASH_READ_SIZE) return 0; - while (iend - ip > HASH_READ_SIZE) { - size_t const remaining = (size_t)(iend - ip); - size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); - const BYTE* const ichunk = ip + chunk; + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); - ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm, tfp); + break; + case ZSTD_dfast: +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; - if (params->ldmParams.enableLdm && ls != NULL) - ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, ¶ms->ldmParams); - - switch(params->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, ichunk, dtlm); - break; - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); - break; - - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (chunk >= HASH_READ_SIZE && ms->dedicatedDictSearch) { - assert(chunk == remaining); /* must load everything in one go */ - ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, ichunk-HASH_READ_SIZE); - } else if (chunk >= HASH_READ_SIZE) { - ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); } - break; - - case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - if (chunk >= HASH_READ_SIZE) - ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); - break; - - default: - assert(0); /* not possible : not a valid strategy id */ } +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; - ip = ichunk; + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + DEBUGLOG(4, "Fill %u bytes into the Binary Tree", (unsigned)srcSize); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + default: + assert(0); /* not possible : not a valid strategy id */ } ms->nextToUpdate = (U32)(iend - ms->window.base); @@ -17681,20 +24827,19 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { unsigned maxSymbolValue = 255; unsigned hasZeroWeights = 1; size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, - dictEnd-dictPtr, &hasZeroWeights); + (size_t)(dictEnd-dictPtr), &hasZeroWeights); /* We only set the loaded table as valid if it contains all non-zero * weights. Otherwise, we set it to check */ - if (!hasZeroWeights) + if (!hasZeroWeights && maxSymbolValue == 255) bs->entropy.huf.repeatMode = HUF_repeat_valid; RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); - RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); dictPtr += hufHeaderSize; } { unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); /* fill all offset symbols to avoid garbage at end of table */ @@ -17709,7 +24854,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; - size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( @@ -17723,7 +24868,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; - size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( @@ -17757,7 +24902,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); } } } - return dictPtr - (const BYTE*)dict; + return (size_t)(dictPtr - (const BYTE*)dict); } /* Dictionary format : @@ -17770,18 +24915,18 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, * dictSize supposed >= 8 */ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, - ZSTD_matchState_t* ms, + ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; size_t dictID; size_t eSize; - ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); @@ -17794,7 +24939,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( - ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } return dictID; } @@ -17803,13 +24948,14 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, * @return : dictID, or an error code */ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, - ZSTD_matchState_t* ms, + ZSTD_MatchState_t* ms, ldmState_t* ls, ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); @@ -17822,13 +24968,13 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent( - ms, ls, ws, params, dict, dictSize, dtlm); + ms, ls, ws, params, dict, dictSize, dtlm, tfp); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); assert(0); /* impossible */ @@ -17836,13 +24982,14 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary( - bs, ms, ws, params, dict, dictSize, dtlm, workspace); + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) /*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, @@ -17852,8 +24999,9 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; #if ZSTD_TRACE - cctx->traceCtx = ZSTD_trace_compress_begin(cctx); + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; #endif DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ @@ -17869,22 +25017,23 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - cctx->entropyWorkspace) + ZSTD_tfp_forCCtx, cctx->tmpWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, cctx->entropyWorkspace); + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; - cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize; + cctx->dictContentSize = dictContentSize; } return 0; } @@ -17921,11 +25070,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, &cctxParams, pledgedSrcSize); } -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_CCtx_params cctxParams; - { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); } DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); @@ -17933,9 +25082,15 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) { - return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); } @@ -17946,14 +25101,13 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - size_t fhSize = 0; DEBUGLOG(4, "ZSTD_writeEpilogue"); RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + size_t fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); dstCapacity -= fhSize; op += fhSize; @@ -17963,8 +25117,9 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->stage != ZSTDcs_ending) { /* write one last empty block, make it the "last" block */ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); - MEM_writeLE32(op, cBlockHeader24); + ZSTD_STATIC_ASSERT(ZSTD_BLOCKHEADERSIZE == 3); + RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE24(op, cBlockHeader24); op += ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize; } @@ -17978,13 +25133,13 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ - return op-ostart; + return (size_t)(op-ostart); } void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) { #if ZSTD_TRACE - if (cctx->traceCtx) { + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); @@ -18005,9 +25160,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) #endif } -size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t endResult; size_t const cSize = ZSTD_compressContinue_internal(cctx, @@ -18031,21 +25186,28 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, return cSize + endResult; } +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params) { - ZSTD_CCtx_params cctxParams; DEBUGLOG(4, "ZSTD_compress_advanced"); FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - &cctxParams); + &cctx->simpleApiParams); } /* Internal */ @@ -18060,7 +25222,7 @@ size_t ZSTD_compress_advanced_internal( FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered) , ""); - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); } size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, @@ -18069,14 +25231,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtx_params cctxParams; { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); assert(params.fParams.contentSizeFlag == 1); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); } DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -18120,7 +25281,10 @@ size_t ZSTD_estimateCDictSize_advanced( DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) - + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } @@ -18151,9 +25315,6 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(params.cParams)); cdict->matchState.cParams = params.cParams; cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; - if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) { - cdict->matchState.dedicatedDictSearch = 0; - } if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictContent = dictBuffer; } else { @@ -18174,6 +25335,7 @@ static size_t ZSTD_initCDict_internal( &cdict->matchState, &cdict->workspace, ¶ms.cParams, + params.useRowMatchFinder, ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict), ""); @@ -18185,7 +25347,7 @@ static size_t ZSTD_initCDict_internal( { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -18195,16 +25357,21 @@ static size_t ZSTD_initCDict_internal( return 0; } -static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_compressionParameters cParams, ZSTD_customMem customMem) +static ZSTD_CDict* +ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_ParamSwitch_e useRowMatchFinder, + int enableDedicatedDictSearch, + ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (dictSize=%u)", (unsigned)dictSize); { size_t const workspaceSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + - ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); @@ -18223,7 +25390,7 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ - + cdict->useRowMatchFinder = useRowMatchFinder; return cdict; } } @@ -18236,6 +25403,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, { ZSTD_CCtx_params cctxParams; ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + DEBUGLOG(3, "ZSTD_createCDict_advanced, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType); ZSTD_CCtxParams_init(&cctxParams, 0); cctxParams.cParams = cParams; cctxParams.customMem = customMem; @@ -18245,7 +25413,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, &cctxParams, customMem); } -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( +ZSTD_CDict* ZSTD_createCDict_advanced2( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -18256,7 +25424,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( ZSTD_compressionParameters cParams; ZSTD_CDict* cdict; - DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (cctxParams.enableDedicatedDictSearch) { @@ -18275,13 +25443,16 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); } + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DedicatedDictSearch=%u", cctxParams.enableDedicatedDictSearch); cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); cdict = ZSTD_createCDict_advanced_internal(dictSize, dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, customMem); - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, dictLoadMethod, dictContentType, cctxParams) )) { @@ -18335,7 +25506,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) * workspaceSize: Use ZSTD_estimateCDictSize() * to determine how large workspace must be. * cParams : use ZSTD_getCParams() to transform a compression level - * into its relevants cParams. + * into its relevant cParams. * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) * Note : there is no corresponding "free" function. * Since workspace was allocated externally, it must be freed externally. @@ -18347,7 +25518,9 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) { - size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) @@ -18356,6 +25529,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_CDict* cdict; ZSTD_CCtx_params params; + DEBUGLOG(4, "ZSTD_initStaticCDict (dictSize==%u)", (unsigned)dictSize); if ((size_t)workspace & 7) return NULL; /* 8-aligned */ { @@ -18366,12 +25540,13 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_cwksp_move(&cdict->workspace, &ws); } - DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", - (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; ZSTD_CCtxParams_init(¶ms, 0); params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, @@ -18398,15 +25573,15 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) return cdict->dictID; } - -/* ZSTD_compressBegin_usingCDict_advanced() : - * cdict must be != NULL */ -size_t ZSTD_compressBegin_usingCDict_advanced( +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { ZSTD_CCtx_params cctxParams; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); /* Initialize the cctxParams from the cdict */ { @@ -18438,23 +25613,51 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTDb_not_buffered); } -/* ZSTD_compressBegin_usingCDict() : - * pledgedSrcSize=0 means "unknown" - * if pledgedSrcSize>0, it will enable contentSizeFlag */ -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { - ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); - return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); } +/* ZSTD_compressBegin_usingCDict() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } /*! ZSTD_compress_usingCDict() : @@ -18468,7 +25671,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } @@ -18509,7 +25712,7 @@ size_t ZSTD_CStreamOutSize(void) return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +static ZSTD_CParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) { if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) return ZSTD_cpm_attachDict; @@ -18640,30 +25843,41 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) { - size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; - if (hintInSize==0) hintInSize = cctx->blockSize; - return hintInSize; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSizeMax - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSizeMax; + return hintInSize; + } } /** ZSTD_compressStream_generic(): * internal function for all *compressStream*() variants - * non-static, because can be called from zstdmt_compress.c - * @return : hint size for next input */ + * @return : hint size for next input to complete ongoing block */ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective const flushMode) { - const char* const istart = (const char*)input->src; - const char* const iend = input->size != 0 ? istart + input->size : istart; - const char* ip = input->pos != 0 ? istart + input->pos : istart; - char* const ostart = (char*)output->dst; - char* const oend = output->size != 0 ? ostart + output->size : ostart; - char* op = output->pos != 0 ? ostart + output->pos : ostart; + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; U32 someMoreWork = 1; /* check expectations */ - DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + if (ip) ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { assert(zcs->inBuff != NULL); assert(zcs->inBuffSize > 0); @@ -18672,8 +25886,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, assert(zcs->outBuff != NULL); assert(zcs->outBuffSize > 0); } - assert(output->pos <= output->size); + if (input->src == NULL) assert(input->size == 0); assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); assert((U32)flushMode <= (U32)ZSTD_e_end); while (someMoreWork) { @@ -18684,12 +25900,13 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_load: if ( (flushMode == ZSTD_e_end) - && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + && ( (size_t)(oend-op) >= ZSTD_compressBound((size_t)(iend-ip)) /* Enough output space */ || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ && (zcs->inBuffPos == 0) ) { /* shortcut to compression pass directly into output buffer */ - size_t const cSize = ZSTD_compressEnd(zcs, - op, oend-op, ip, iend-ip); + size_t const cSize = ZSTD_compressEnd_public(zcs, + op, (size_t)(oend-op), + ip, (size_t)(iend-ip)); DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); ip = iend; @@ -18703,10 +25920,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; size_t const loaded = ZSTD_limitCopy( zcs->inBuff + zcs->inBuffPos, toLoad, - ip, iend-ip); + ip, (size_t)(iend-ip)); zcs->inBuffPos += loaded; - if (loaded != 0) - ip += loaded; + if (ip) ip += loaded; if ( (flushMode == ZSTD_e_continue) && (zcs->inBuffPos < zcs->inBuffTarget) ) { /* not enough input to fill full block : stop here */ @@ -18717,16 +25933,29 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* empty */ someMoreWork = 0; break; } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSizeMax) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } } /* compress current block (note : this stage cannot be stopped in the middle) */ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); void* cDst; size_t cSize; - size_t oSize = oend-op; - size_t const iSize = inputBuffered - ? zcs->inBuffPos - zcs->inToCompress - : MIN((size_t)(iend - ip), zcs->blockSize); + size_t oSize = (size_t)(oend-op); + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSizeMax); if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) cDst = op; /* compress into output buffer, to skip flush stage */ else @@ -18734,34 +25963,31 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (inputBuffered) { unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, + ZSTD_compressEnd_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, + ZSTD_compressContinue_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; /* prepare next block */ - zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSizeMax; if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSizeMax; DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); if (!lastBlock) assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; - } else { - unsigned const lastBlock = (ip + iSize == iend); - assert(flushMode == ZSTD_e_end /* Already validated */); + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); /* Consume the input prior to error checking to mirror buffered mode. */ - if (iSize > 0) - ip += iSize; + if (ip) ip += iSize; FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; - if (lastBlock) - assert(ip == iend); + if (lastBlock) assert(ip == iend); } if (cDst == op) { /* no need to flush */ op += cSize; @@ -18776,7 +26002,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->outBuffFlushedSize = 0; zcs->streamStage = zcss_flush; /* pass-through to flush stage */ } - /* fall-through */ + ZSTD_FALLTHROUGH; case zcss_flush: DEBUGLOG(5, "flush stage"); assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); @@ -18810,8 +26036,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, } } - input->pos = ip - istart; - output->pos = op - ostart; + input->pos = (size_t)(ip - istart); + output->pos = (size_t)(op - ostart); if (zcs->frameEnded) return 0; return ZSTD_nextInputSizeHint(zcs); } @@ -18837,8 +26063,10 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf /* After a compression call set the expected input/output buffer. * This is validated at the start of the next compression call. */ -static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) { + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { cctx->expectedInBuffer = *input; } @@ -18857,55 +26085,73 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, { if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { ZSTD_inBuffer const expect = cctx->expectedInBuffer; - if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); - if (endOp != ZSTD_e_end) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); } + (void)endOp; if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { size_t const outBufferSize = output->size - output->pos; if (cctx->expectedOutBufferSize != outBufferSize) - RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); } return 0; } +/* + * If @endOp == ZSTD_e_end, @inSize becomes pledgedSrcSize. + * Otherwise, it's ignored. + * @return: 0 on success, or a ZSTD_error code otherwise. + */ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, ZSTD_EndDirective endOp, - size_t inSize) { + size_t inSize) +{ ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ - if (cctx->cdict) - params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */ - DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); - if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ - { - size_t const dictSize = prefixDict.dict + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_CCtx_init_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict ? prefixDict.dictSize : (cctx->cdict ? cctx->cdict->dictContentSize : 0); - ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + ZSTD_CParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); params.cParams = ZSTD_getCParamsFromCCtxParams( ¶ms, cctx->pledgedSrcSizePlusOne-1, dictSize, mode); } - if (ZSTD_CParams_shouldEnableLdm(¶ms.cParams)) { - /* Enable LDM by default for optimal parser and window size >= 128MB */ - DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); - params.ldmParams.enableLdm = 1; - } + params.postBlockSplitter = ZSTD_resolveBlockSplitterMode(params.postBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); #ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(¶ms) && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ } if (params.nbWorkers > 0) { -#if ZSTD_TRACE - cctx->traceCtx = ZSTD_trace_compress_begin(cctx); -#endif +# if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +# endif /* mt context creation */ if (cctx->mtctx == NULL) { DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", @@ -18926,7 +26172,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, cctx->streamStage = zcss_load; cctx->appliedParams = params; } else -#endif +#endif /* ZSTD_MULTITHREAD */ { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -18941,7 +26187,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, /* for small input: avoid automatic flush on reaching end of block, since * it would require to add a 3-bytes null block to end frame */ - cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + cctx->inBuffTarget = cctx->blockSizeMax + (cctx->blockSizeMax == pledgedSrcSize); } else { cctx->inBuffTarget = 0; } @@ -18952,6 +26198,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, return 0; } +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, @@ -18966,8 +26214,27 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); - ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ } /* end of transparent initialization stage */ @@ -18980,6 +26247,13 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } for (;;) { size_t const ipos = input->pos; size_t const opos = output->pos; @@ -19018,7 +26292,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_setBufferExpectations(cctx, output, input); return flushMin; } -#endif +#endif /* ZSTD_MULTITHREAD */ FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); DEBUGLOG(5, "completed ZSTD_compressStream2"); ZSTD_setBufferExpectations(cctx, output, input); @@ -19031,13 +26305,20 @@ size_t ZSTD_compressStream2_simpleArgs ( const void* src, size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } size_t ZSTD_compress2(ZSTD_CCtx* cctx, @@ -19060,6 +26341,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, /* Reset to the original values. */ cctx->requestedParams.inBufferMode = originalInBufferMode; cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); @@ -19070,64 +26352,66 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, } } -typedef struct { - U32 idx; /* Index in array of ZSTD_Sequence */ - U32 posInSequence; /* Position within sequence at idx */ - size_t posInSrc; /* Number of bytes given by sequences provided so far */ -} ZSTD_sequencePosition; - -/* Returns a ZSTD error code if sequence is not valid */ -static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, - size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { - size_t offsetBound; - U32 windowSize = 1 << windowLog; - /* posInSrc represents the amount of data the the decoder would decode up to this point. +/* ZSTD_validateSequence() : + * @offBase : must use the format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) +{ + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. * As long as the amount of data decoded is less than or equal to window size, offsets may be * larger than the total length of output decoded in order to reference the dict, even larger than * window size. After output surpasses windowSize, we're limited to windowSize offsets again. */ - offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; - RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); - RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offBase > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); return 0; } /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ -static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { - U32 offCode = rawOffset + ZSTD_REP_MOVE; - U32 repCode = 0; +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); if (!ll0 && rawOffset == rep[0]) { - repCode = 1; + offBase = REPCODE1_TO_OFFBASE; } else if (rawOffset == rep[1]) { - repCode = 2 - ll0; + offBase = REPCODE_TO_OFFBASE(2 - ll0); } else if (rawOffset == rep[2]) { - repCode = 3 - ll0; + offBase = REPCODE_TO_OFFBASE(3 - ll0); } else if (ll0 && rawOffset == rep[0] - 1) { - repCode = 3; + offBase = REPCODE3_TO_OFFBASE; } - if (repCode) { - /* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */ - offCode = repCode - 1; - } - return offCode; + return offBase; } -/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of - * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. +/* This function scans through an array of ZSTD_Sequence, + * storing the sequences it reads, until it reaches a block delimiter. + * Note that the block delimiter includes the last literals of the block. + * @blockSize must be == sum(sequence_lengths). + * @returns @blockSize on success, and a ZSTD_error otherwise. */ -static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) { +static size_t +ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch) +{ U32 idx = seqPos->idx; + U32 const startIdx = idx; BYTE const* ip = (BYTE const*)(src); const BYTE* const iend = ip + blockSize; - repcodes_t updatedRepcodes; + Repcodes_t updatedRepcodes; U32 dictSize; - U32 litLength; - U32 matchLength; - U32 ll0; - U32 offCode; + + DEBUGLOG(5, "ZSTD_transferSequences_wBlockDelim (blockSize = %zu)", blockSize); if (cctx->cdict) { dictSize = (U32)cctx->cdict->dictContentSize; @@ -19136,28 +26420,60 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS } else { dictSize = 0; } - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); - for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { - litLength = inSeqs[idx].litLength; - matchLength = inSeqs[idx].matchLength; - ll0 = litLength == 0; - offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const matchLength = inSeqs[idx].matchLength; + U32 offBase; - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, + seqPos->posInSrc, cctx->appliedParams.cParams.windowLog, dictSize, - cctx->appliedParams.cParams.minMatch), + ZSTD_hasExtSeqProd(&cctx->appliedParams)), "Sequence validation failed"); } - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; } - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + RETURN_ERROR_IF(idx == inSeqsSize, externalSequences_invalid, "Block delimiter not found."); + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); if (inSeqs[idx].litLength) { DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); @@ -19165,38 +26481,42 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS ip += inSeqs[idx].litLength; seqPos->posInSrc += inSeqs[idx].litLength; } - RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); seqPos->idx = idx+1; - return 0; + return blockSize; } -/* Returns the number of bytes to move the current read position back by. Only non-zero - * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something - * went wrong. +/* + * This function attempts to scan through @blockSize bytes in @src + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. * - * This function will attempt to scan through blockSize bytes represented by the sequences - * in inSeqs, storing any (partial) sequences. + * Occasionally, we may want to reduce the actual number of bytes consumed from @src + * to avoid splitting a match, notably if it would produce a match smaller than MINMATCH. * - * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to - * avoid splitting a match, or to avoid splitting a match such that it would produce a match - * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + * @returns the number of bytes consumed from @src, necessarily <= @blockSize. + * Otherwise, it may return a ZSTD error if something went wrong. */ -static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) { +static size_t +ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch) +{ U32 idx = seqPos->idx; U32 startPosInSequence = seqPos->posInSequence; U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; size_t dictSize; - BYTE const* ip = (BYTE const*)(src); - BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ - repcodes_t updatedRepcodes; + const BYTE* const istart = (const BYTE*)(src); + const BYTE* ip = istart; + const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + Repcodes_t updatedRepcodes; U32 bytesAdjustment = 0; U32 finalMatchSplit = 0; - U32 litLength; - U32 matchLength; - U32 rawOffset; - U32 offCode; + + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; if (cctx->cdict) { dictSize = cctx->cdict->dictContentSize; @@ -19205,14 +26525,15 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq } else { dictSize = 0; } - DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { const ZSTD_Sequence currSeq = inSeqs[idx]; - litLength = currSeq.litLength; - matchLength = currSeq.matchLength; - rawOffset = currSeq.offset; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offBase; /* Modify the sequence depending on where endPosInSequence lies */ if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { @@ -19226,7 +26547,6 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq /* Move to the next sequence */ endPosInSequence -= currSeq.litLength + currSeq.matchLength; startPosInSequence = 0; - idx++; } else { /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence does not reach the end of the match. So, we have to split the sequence */ @@ -19265,76 +26585,131 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq } } /* Check if this offset can be represented with a repcode */ - { U32 ll0 = (litLength == 0); - offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); - updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + { U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); } if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, - cctx->appliedParams.cParams.windowLog, dictSize, - cctx->appliedParams.cParams.minMatch), + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), "Sequence validation failed"); } - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ } DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); seqPos->idx = idx; seqPos->posInSequence = endPosInSequence; - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); iend -= bytesAdjustment; if (ip != iend) { /* Store any last literals */ - U32 lastLLSize = (U32)(iend - ip); + U32 const lastLLSize = (U32)(iend - ip); assert(ip <= iend); DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); seqPos->posInSrc += lastLLSize; } - return bytesAdjustment; + return (size_t)(iend-istart); } -typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize); -static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { - ZSTD_sequenceCopier sequenceCopier = NULL; - assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); - if (mode == ZSTD_sf_explicitBlockDelimiters) { - return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; - } else if (mode == ZSTD_sf_noBlockDelimiters) { - return ZSTD_copySequencesToSeqStoreNoBlockDelim; - } - assert(sequenceCopier != NULL); - return sequenceCopier; -} - -/* Compress, block-by-block, all of the sequences given. - * - * Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error. +/* @seqPos represents a position within @inSeqs, + * it is read and updated by this function, + * once the goal to produce a block of size @blockSize is reached. + * @return: nb of bytes consumed from @src, necessarily <= @blockSize. */ -static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize) { - size_t cSize = 0; - U32 lastBlock; - size_t blockSize; - size_t compressedSeqsSize; - size_t remaining = srcSize; - ZSTD_sequencePosition seqPos = {0, 0, 0}; +typedef size_t (*ZSTD_SequenceCopier_f)(ZSTD_CCtx* cctx, + ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_ParamSwitch_e externalRepSearch); - BYTE const* ip = (BYTE const*)src; +static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mode) +{ + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, (int)mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_transferSequences_wBlockDelim; + } + assert(mode == ZSTD_sf_noBlockDelimiters); + return ZSTD_transferSequences_noDelim; +} + +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_SequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +static size_t determine_blockSize(ZSTD_SequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + ZSTD_SequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) { + /* Note: more a "target" block size */ + return MIN(remaining, blockSize); + } + assert(mode == ZSTD_sf_explicitBlockDelimiters); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + +/* Compress all provided sequences, block-by-block. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + size_t remaining = srcSize; + ZSTD_SequencePosition seqPos = {0, 0, 0}; + + const BYTE* ip = (BYTE const*)src; BYTE* op = (BYTE*)dst; - ZSTD_sequenceCopier sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); /* Special case: empty frame */ @@ -19348,22 +26723,29 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, } while (remaining) { + size_t compressedSeqsSize; size_t cBlockSize; - size_t additionalByteAdjustment; - lastBlock = remaining <= cctx->blockSize; - blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSizeMax, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); ZSTD_resetSeqStore(&cctx->seqStore); - DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); - additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); - FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); - blockSize -= additionalByteAdjustment; + blockSize = sequenceCopier(cctx, + &seqPos, inSeqs, inSeqsSize, + ip, blockSize, + cctx->appliedParams.searchForExternalRepcodes); + FORWARD_IF_ERROR(blockSize, "Bad sequence copy"); /* If blocks are too small, emit as a nocompress block */ - if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + DEBUGLOG(5, "Block too small (%zu): data remains uncompressed: cSize=%zu", blockSize, cBlockSize); cSize += cBlockSize; ip += blockSize; op += cBlockSize; @@ -19372,39 +26754,40 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, continue; } - compressedSeqsSize = ZSTD_entropyCompressSequences(&cctx->seqStore, + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, &cctx->appliedParams, op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, blockSize, - cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, cctx->bmi2); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); - DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); if (!cctx->isFirstBlock && ZSTD_maybeRLE(&cctx->seqStore) && - ZSTD_isRLE((BYTE const*)src, srcSize)) { - /* We don't want to emit our first block as a RLE even if it qualifies because - * doing so will cause the decoder (cli only) to throw a "should consume all input error." - * This is only an issue for zstd <= v1.4.3 - */ + ZSTD_isRLE(ip, blockSize)) { + /* Note: don't emit the first block as RLE even if it qualifies because + * doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error + * "should consume all input error." + */ compressedSeqsSize = 1; } if (compressedSeqsSize == 0) { /* ZSTD_noCompressBlock writes the block header as well */ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); } else if (compressedSeqsSize == 1) { cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); - DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); } else { U32 cBlockHeader; /* Error checking and repcodes update */ - ZSTD_confirmRepcodesAndEntropyTables(cctx); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; @@ -19412,11 +26795,10 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); MEM_writeLE24(op, cBlockHeader); cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; - DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); } cSize += cBlockSize; - DEBUGLOG(4, "cSize running total: %zu", cSize); if (lastBlock) { break; @@ -19427,40 +26809,50 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, dstCapacity -= cBlockSize; cctx->isFirstBlock = 0; } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); } + DEBUGLOG(4, "cSize final total: %zu", cSize); return cSize; } -size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize) { + const void* src, size_t srcSize) +{ BYTE* op = (BYTE*)dst; size_t cSize = 0; - size_t compressedBlocksSize = 0; - size_t frameHeaderSize = 0; /* Transparent initialization stage, same as compressStream2() */ - DEBUGLOG(3, "ZSTD_compressSequences()"); + DEBUGLOG(4, "ZSTD_compressSequences (nbSeqs=%zu,dstCapacity=%zu)", inSeqsSize, dstCapacity); assert(cctx != NULL); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ - frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); - op += frameHeaderSize; - dstCapacity -= frameHeaderSize; - cSize += frameHeaderSize; + { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, + &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + assert(frameHeaderSize <= dstCapacity); + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + } if (cctx->appliedParams.fParams.checksumFlag && srcSize) { XXH64_update(&cctx->xxhState, src, srcSize); } - /* cSize includes block header size and compressed sequences size */ - compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + + /* Now generate compressed blocks */ + { size_t const cBlocksSize = ZSTD_compressSequences_internal(cctx, op, dstCapacity, inSeqs, inSeqsSize, src, srcSize); - FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); - cSize += compressedBlocksSize; - dstCapacity -= compressedBlocksSize; + FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!"); + cSize += cBlocksSize; + assert(cBlocksSize <= dstCapacity); + dstCapacity -= cBlocksSize; + } + /* Complete with frame checksum, if needed */ if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); @@ -19469,26 +26861,557 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci cSize += 4; } - DEBUGLOG(3, "Final compressed size: %zu", cSize); + DEBUGLOG(4, "Final compressed size: %zu", cSize); + return cSize; +} + + +#if defined(__AVX2__) + +#include /* AVX2 intrinsics */ + +/* + * Convert 2 sequences per iteration, using AVX2 intrinsics: + * - offset -> offBase = offset + 2 + * - litLength -> (U16) litLength + * - matchLength -> (U16)(matchLength - 3) + * - rep is ignored + * Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]). + * + * At the end, instead of extracting two __m128i, + * we use _mm256_permute4x64_epi64(..., 0xE8) to move lane2 into lane1, + * then store the lower 16 bytes in one go. + * + * @returns 0 on succes, with no long length detected + * @returns > 0 if there is one long length (> 65535), + * indicating the position, and type. + */ +static size_t convertSequences_noRepcodes( + SeqDef* dstSeqs, + const ZSTD_Sequence* inSeqs, + size_t nbSequences) +{ + /* + * addition: + * For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0) + */ + const __m256i addition = _mm256_setr_epi32( + ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */ + ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */ + ); + + /* limit: check if there is a long length */ + const __m256i limit = _mm256_set1_epi32(65535); + + /* + * shuffle mask for byte-level rearrangement in each 128-bit half: + * + * Input layout (after addition) per 128-bit half: + * [ offset+2 (4 bytes) | litLength (4 bytes) | matchLength (4 bytes) | rep (4 bytes) ] + * We only need: + * offBase (4 bytes) = offset+2 + * litLength (2 bytes) = low 2 bytes of litLength + * mlBase (2 bytes) = low 2 bytes of (matchLength) + * => Bytes [0..3, 4..5, 8..9], zero the rest. + */ + const __m256i mask = _mm256_setr_epi8( + /* For the lower 128 bits => sequence i */ + 0, 1, 2, 3, /* offset+2 */ + 4, 5, /* litLength (16 bits) */ + 8, 9, /* matchLength (16 bits) */ + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + + /* For the upper 128 bits => sequence i+1 */ + 16,17,18,19, /* offset+2 */ + 20,21, /* litLength */ + 24,25, /* matchLength */ + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, + (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80 + ); + + /* + * Next, we'll use _mm256_permute4x64_epi64(vshf, 0xE8). + * Explanation of 0xE8 = 11101000b => [lane0, lane2, lane2, lane3]. + * So the lower 128 bits become [lane0, lane2] => combining seq0 and seq1. + */ +#define PERM_LANE_0X_E8 0xE8 /* [0,2,2,3] in lane indices */ + + size_t longLen = 0, i = 0; + + /* AVX permutation depends on the specific definition of target structures */ + ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8); + ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4); + ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6); + + /* Process 2 sequences per loop iteration */ + for (; i + 1 < nbSequences; i += 2) { + /* Load 2 ZSTD_Sequence (32 bytes) */ + __m256i vin = _mm256_loadu_si256((const __m256i*)(const void*)&inSeqs[i]); + + /* Add {2, 0, -3, 0} in each 128-bit half */ + __m256i vadd = _mm256_add_epi32(vin, addition); + + /* Check for long length */ + __m256i ll_cmp = _mm256_cmpgt_epi32(vadd, limit); /* 0xFFFFFFFF for element > 65535 */ + int ll_res = _mm256_movemask_epi8(ll_cmp); + + /* Shuffle bytes so each half gives us the 8 bytes we need */ + __m256i vshf = _mm256_shuffle_epi8(vadd, mask); + /* + * Now: + * Lane0 = seq0's 8 bytes + * Lane1 = 0 + * Lane2 = seq1's 8 bytes + * Lane3 = 0 + */ + + /* Permute 64-bit lanes => move Lane2 down into Lane1. */ + __m256i vperm = _mm256_permute4x64_epi64(vshf, PERM_LANE_0X_E8); + /* + * Now the lower 16 bytes (Lane0+Lane1) = [seq0, seq1]. + * The upper 16 bytes are [Lane2, Lane3] = [seq1, 0], but we won't use them. + */ + + /* Store only the lower 16 bytes => 2 SeqDef (8 bytes each) */ + _mm_storeu_si128((__m128i *)(void*)&dstSeqs[i], _mm256_castsi256_si128(vperm)); + /* + * This writes out 16 bytes total: + * - offset 0..7 => seq0 (offBase, litLength, mlBase) + * - offset 8..15 => seq1 (offBase, litLength, mlBase) + */ + + /* check (unlikely) long lengths > 65535 + * indices for lengths correspond to bits [4..7], [8..11], [20..23], [24..27] + * => combined mask = 0x0FF00FF0 + */ + if (UNLIKELY((ll_res & 0x0FF00FF0) != 0)) { + /* long length detected: let's figure out which one*/ + if (inSeqs[i].matchLength > 65535+MINMATCH) { + assert(longLen == 0); + longLen = i + 1; + } + if (inSeqs[i].litLength > 65535) { + assert(longLen == 0); + longLen = i + nbSequences + 1; + } + if (inSeqs[i+1].matchLength > 65535+MINMATCH) { + assert(longLen == 0); + longLen = i + 1 + 1; + } + if (inSeqs[i+1].litLength > 65535) { + assert(longLen == 0); + longLen = i + 1 + nbSequences + 1; + } + } + } + + /* Handle leftover if @nbSequences is odd */ + if (i < nbSequences) { + /* process last sequence */ + assert(i == nbSequences - 1); + dstSeqs[i].offBase = OFFSET_TO_OFFBASE(inSeqs[i].offset); + dstSeqs[i].litLength = (U16)inSeqs[i].litLength; + dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH); + /* check (unlikely) long lengths > 65535 */ + if (UNLIKELY(inSeqs[i].matchLength > 65535+MINMATCH)) { + assert(longLen == 0); + longLen = i + 1; + } + if (UNLIKELY(inSeqs[i].litLength > 65535)) { + assert(longLen == 0); + longLen = i + nbSequences + 1; + } + } + + return longLen; +} + +/* the vector implementation could also be ported to SSSE3, + * but since this implementation is targeting modern systems (>= Sapphire Rapid), + * it's not useful to develop and maintain code for older pre-AVX2 platforms */ + +#else /* no AVX2 */ + +static size_t convertSequences_noRepcodes( + SeqDef* dstSeqs, + const ZSTD_Sequence* inSeqs, + size_t nbSequences) +{ + size_t longLen = 0; + size_t n; + for (n=0; n 65535 */ + if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) { + assert(longLen == 0); + longLen = n + 1; + } + if (UNLIKELY(inSeqs[n].litLength > 65535)) { + assert(longLen == 0); + longLen = n + nbSequences + 1; + } + } + return longLen; +} + +#endif + +/* + * Precondition: Sequences must end on an explicit Block Delimiter + * @return: 0 on success, or an error code. + * Note: Sequence validation functionality has been disabled (removed). + * This is helpful to generate a lean main pipeline, improving performance. + * It may be re-inserted later. + */ +size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, + const ZSTD_Sequence* const inSeqs, size_t nbSequences, + int repcodeResolution) +{ + Repcodes_t updatedRepcodes; + size_t seqNb = 0; + + DEBUGLOG(5, "ZSTD_convertBlockSequences (nbSequences = %zu)", nbSequences); + + RETURN_ERROR_IF(nbSequences >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); + + /* check end condition */ + assert(nbSequences >= 1); + assert(inSeqs[nbSequences-1].matchLength == 0); + assert(inSeqs[nbSequences-1].offset == 0); + + /* Convert Sequences from public format to internal format */ + if (!repcodeResolution) { + size_t const longl = convertSequences_noRepcodes(cctx->seqStore.sequencesStart, inSeqs, nbSequences-1); + cctx->seqStore.sequences = cctx->seqStore.sequencesStart + nbSequences-1; + if (longl) { + DEBUGLOG(5, "long length"); + assert(cctx->seqStore.longLengthType == ZSTD_llt_none); + if (longl <= nbSequences-1) { + DEBUGLOG(5, "long match length detected at pos %zu", longl-1); + cctx->seqStore.longLengthType = ZSTD_llt_matchLength; + cctx->seqStore.longLengthPos = (U32)(longl-1); + } else { + DEBUGLOG(5, "long literals length detected at pos %zu", longl-nbSequences); + assert(longl <= 2* (nbSequences-1)); + cctx->seqStore.longLengthType = ZSTD_llt_literalLength; + cctx->seqStore.longLengthPos = (U32)(longl-(nbSequences-1)-1); + } + } + } else { + for (seqNb = 0; seqNb < nbSequences - 1 ; seqNb++) { + U32 const litLength = inSeqs[seqNb].litLength; + U32 const matchLength = inSeqs[seqNb].matchLength; + U32 const ll0 = (litLength == 0); + U32 const offBase = ZSTD_finalizeOffBase(inSeqs[seqNb].offset, updatedRepcodes.rep, ll0); + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + if (!repcodeResolution && nbSequences > 1) { + U32* const rep = updatedRepcodes.rep; + + if (nbSequences >= 4) { + U32 lastSeqIdx = (U32)nbSequences - 2; /* index of last full sequence */ + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (nbSequences == 3) { + rep[2] = rep[0]; + rep[1] = inSeqs[0].offset; + rep[0] = inSeqs[1].offset; + } else { + assert(nbSequences == 2); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[0].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); + + return 0; +} + +#if defined(ZSTD_ARCH_X86_AVX2) + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs) +{ + size_t i; + __m256i const zeroVec = _mm256_setzero_si256(); + __m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */ + ZSTD_ALIGNED(32) U32 tmp[8]; /* temporary buffer for reduction */ + size_t mSum = 0, lSum = 0; + ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16); + + /* Process 2 structs (32 bytes) at a time */ + for (i = 0; i + 2 <= nbSeqs; i += 2) { + /* Load two consecutive ZSTD_Sequence (8×4 = 32 bytes) */ + __m256i data = _mm256_loadu_si256((const __m256i*)(const void*)&seqs[i]); + /* check end of block signal */ + __m256i cmp = _mm256_cmpeq_epi32(data, zeroVec); + int cmp_res = _mm256_movemask_epi8(cmp); + /* indices for match lengths correspond to bits [8..11], [24..27] + * => combined mask = 0x0F000F00 */ + ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8); + if (cmp_res & 0x0F000F00) break; + /* Accumulate in sumVec */ + sumVec = _mm256_add_epi32(sumVec, data); + } + + /* Horizontal reduction */ + _mm256_store_si256((__m256i*)tmp, sumVec); + lSum = tmp[1] + tmp[5]; + mSum = tmp[2] + tmp[6]; + + /* Handle the leftover */ + for (; i < nbSeqs; i++) { + lSum += seqs[i].litLength; + mSum += seqs[i].matchLength; + if (seqs[i].matchLength == 0) break; /* end of block */ + } + + if (i==nbSeqs) { + /* reaching end of sequences: end of block signal was not present */ + BlockSummary bs; + bs.nbSequences = ERROR(externalSequences_invalid); + return bs; + } + { BlockSummary bs; + bs.nbSequences = i+1; + bs.blockSize = lSum + mSum; + bs.litSize = lSum; + return bs; + } +} + +#else + +BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs) +{ + size_t totalMatchSize = 0; + size_t litSize = 0; + size_t n; + assert(seqs); + for (n=0; nappliedParams.searchForExternalRepcodes == ZSTD_ps_enable); + assert(cctx->appliedParams.searchForExternalRepcodes != ZSTD_ps_auto); + + DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals_internal: nbSeqs=%zu, litSize=%zu", nbSequences, litSize); + RETURN_ERROR_IF(nbSequences == 0, externalSequences_invalid, "Requires at least 1 end-of-block"); + + /* Special case: empty frame */ + if ((nbSequences == 1) && (inSeqs[0].litLength == 0)) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE24(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (nbSequences) { + size_t compressedSeqsSize, cBlockSize, conversionStatus; + BlockSummary const block = ZSTD_get1BlockSummary(inSeqs, nbSequences); + U32 const lastBlock = (block.nbSequences == nbSequences); + FORWARD_IF_ERROR(block.nbSequences, "Error while trying to determine nb of sequences for a block"); + assert(block.nbSequences <= nbSequences); + RETURN_ERROR_IF(block.litSize > litSize, externalSequences_invalid, "discrepancy: Sequences require more literals than present in buffer"); + ZSTD_resetSeqStore(&cctx->seqStore); + + conversionStatus = ZSTD_convertBlockSequences(cctx, + inSeqs, block.nbSequences, + repcodeResolution); + FORWARD_IF_ERROR(conversionStatus, "Bad sequence conversion"); + inSeqs += block.nbSequences; + nbSequences -= block.nbSequences; + remaining -= block.blockSize; + + /* Note: when blockSize is very small, other variant send it uncompressed. + * Here, we still send the sequences, because we don't have the original source to send it uncompressed. + * One could imagine in theory reproducing the source from the sequences, + * but that's complex and costly memory intensive, and goes against the objectives of this variant. */ + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + + compressedSeqsSize = ZSTD_entropyCompressSeqStore_internal( + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + literals, block.litSize, + &cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + /* note: the spec forbids for any compressed block to be larger than maximum block size */ + if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0; + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + litSize -= block.litSize; + literals = (const char*)literals + block.litSize; + + /* Note: difficult to check source for RLE block when only Literals are provided, + * but it could be considered from analyzing the sequence directly */ + + if (compressedSeqsSize == 0) { + /* Sending uncompressed blocks is out of reach, because the source is not provided. + * In theory, one could use the sequences to regenerate the source, like a decompressor, + * but it's complex, and memory hungry, killing the purpose of this variant. + * Current outcome: generate an error code. + */ + RETURN_ERROR(cannotProduce_uncompressedBlock, "ZSTD_compressSequencesAndLiterals cannot generate an uncompressed block"); + } else { + U32 cBlockHeader; + assert(compressedSeqsSize > 1); /* no RLE */ + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + op += cBlockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); + + if (lastBlock) { + assert(nbSequences == 0); + break; + } + } + + RETURN_ERROR_IF(litSize != 0, externalSequences_invalid, "literals must be entirely and exactly consumed"); + RETURN_ERROR_IF(remaining != 0, externalSequences_invalid, "Sequences must represent a total of exactly srcSize=%zu", srcSize); + DEBUGLOG(4, "cSize final total: %zu", cSize); + return cSize; +} + +size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* literals, size_t litSize, size_t litCapacity, + size_t decompressedSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity); + assert(cctx != NULL); + if (litCapacity < litSize) { + RETURN_ERROR(workSpace_tooSmall, "literals buffer is not large enough: must be at least 8 bytes larger than litSize (risk of read out-of-bound)"); + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, decompressedSize), "CCtx initialization failed"); + + if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) { + RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters"); + } + if (cctx->appliedParams.validateSequences) { + RETURN_ERROR(parameter_unsupported, "This mode is not compatible with Sequence validation"); + } + if (cctx->appliedParams.fParams.checksumFlag) { + RETURN_ERROR(frameParameter_unsupported, "this mode is not compatible with frame checksum"); + } + + /* Begin writing output, starting with frame header */ + { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, + &cctx->appliedParams, decompressedSize, cctx->dictID); + op += frameHeaderSize; + assert(frameHeaderSize <= dstCapacity); + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + } + + /* Now generate compressed blocks */ + { size_t const cBlocksSize = ZSTD_compressSequencesAndLiterals_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + literals, litSize, decompressedSize); + FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!"); + cSize += cBlocksSize; + assert(cBlocksSize <= dstCapacity); + dstCapacity -= cBlocksSize; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); return cSize; } /*====== Finalize ======*/ +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); } - size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); - FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; @@ -19500,11 +27423,31 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) } +/*-===== Pre-defined compression levels =====-*/ +/**** start inlining clevels.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +/**** skipping file: ../zstd.h ****/ + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_MAX_CLEVEL 22 -int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } -int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" - for any srcSize > 256 KB */ @@ -19514,15 +27457,15 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ - { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ - { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ - { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ - { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ - { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ @@ -19539,8 +27482,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ - { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ - { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ @@ -19567,11 +27510,11 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ - { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ - { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ @@ -19613,6 +27556,15 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; + + +#endif /* ZSTD_CLEVELS_H */ +/**** ended inlining clevels.h ****/ + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) { ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); @@ -19637,7 +27589,10 @@ static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const static int ZSTD_dedicatedDictSearch_isSupported( ZSTD_compressionParameters const* cParams) { - return (cParams->strategy >= ZSTD_greedy) && (cParams->strategy <= ZSTD_lazy2); + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); } /** @@ -19655,6 +27610,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams( case ZSTD_lazy: case ZSTD_lazy2: cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } break; case ZSTD_btlazy2: case ZSTD_btopt: @@ -19664,7 +27622,7 @@ static void ZSTD_dedicatedDictSearch_revertCParams( } } -static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { switch (mode) { case ZSTD_cpm_unknown: @@ -19688,8 +27646,8 @@ static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMo * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. * Use dictSize == 0 for unknown or unused. - * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ -static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_CParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); @@ -19703,13 +27661,14 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, else row = compressionLevel; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); /* acceleration factor */ if (compressionLevel < 0) { int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); cp.targetLength = (unsigned)(-clampedCompressionLevel); } /* refine parameters based on srcSize & dictSize */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); } } @@ -19726,7 +27685,9 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { +static ZSTD_parameters +ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) +{ ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); @@ -19740,14 +27701,41 @@ static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned lo * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); } + +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc) +{ + assert(zc != NULL); + ZSTD_CCtxParams_registerSequenceProducer( + &zc->requestedParams, extSeqProdState, extSeqProdFunc + ); +} + +void ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc) +{ + assert(params != NULL); + if (extSeqProdFunc != NULL) { + params->extSeqProdFunc = extSeqProdFunc; + params->extSeqProdState = extSeqProdState; + } else { + params->extSeqProdFunc = NULL; + params->extSeqProdState = NULL; + } +} /**** ended inlining compress/zstd_compress.c ****/ /**** start inlining compress/zstd_double_fast.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -19759,8 +27747,49 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_double_fast.h ****/ +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR -void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; @@ -19791,15 +27820,27 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } } + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } } FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_doubleFast_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, - U32 const mls /* template */, ZSTD_dictMode_e const dictMode) +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) { ZSTD_compressionParameters const* cParams = &ms->cParams; U32* const hashLong = ms->hashTable; @@ -19808,7 +27849,6 @@ size_t ZSTD_compressBlock_doubleFast_generic( const U32 hBitsS = cParams->chainLog; const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); /* presumes that, if there is a dictionary, it must be using Attach mode */ @@ -19817,188 +27857,173 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; + U32 offsetSaved1 = 0, offsetSaved2 = 0; - const ZSTD_matchState_t* const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dictCParams = - dictMode == ZSTD_dictMatchState ? - &dms->cParams : NULL; - const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ? - dms->hashTable : NULL; - const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ? - dms->chainTable : NULL; - const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? - dms->window.dictLimit : 0; - const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? - dms->window.base : NULL; - const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? - dictBase + dictStartIndex : NULL; - const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? - dms->window.nextSrc : NULL; - const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? - prefixLowestIndex - (U32)(dictEnd - dictBase) : - 0; - const U32 dictHBitsL = dictMode == ZSTD_dictMatchState ? - dictCParams->hashLog : hBitsL; - const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? - dictCParams->chainLog : hBitsS; - const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + size_t mLength; + U32 offset; + U32 curr; - DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ - assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ - /* if a dictionary is attached, it must be within window range */ - if (dictMode == ZSTD_dictMatchState) { - assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); - } + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + const BYTE* matchs0_safe; /* matchs0 or safe address */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + /* Array of ~random data, should have low probability of matching data + * we load from here instead of from tables, if matchl0/matchl1 are + * invalid indices. Used to avoid unpredictable branches. */ + const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4}; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); /* init */ - ip += (dictAndPrefixLength == 0); - if (dictMode == ZSTD_noDict) { - U32 const curr = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); - U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; - } - if (dictMode == ZSTD_dictMatchState) { - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; } - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - U32 offset; - size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); - size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); - size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); - U32 const curr = (U32)(ip-base); - U32 const matchIndexL = hashLong[h2]; - U32 matchIndexS = hashSmall[h]; - const BYTE* matchLong = base + matchIndexL; - const BYTE* match = base + matchIndexS; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (dictMode == ZSTD_dictMatchState - && repIndex < prefixLowestIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; - /* check dictMatchState repcode */ - if (dictMode == ZSTD_dictMatchState - && ((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); - goto _match_stored; + if (ip1 > ilimit) { + goto _cleanup; } - /* check noDict repcode */ - if ( dictMode == ZSTD_noDict - && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); - goto _match_stored; - } + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; - if (matchIndexL > prefixLowestIndex) { - /* check prefix long match */ - if (MEM_read64(matchLong) == MEM_read64(ip)) { - mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; - offset = (U32)(ip-matchLong); - while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - goto _match_found; + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; } - } else if (dictMode == ZSTD_dictMatchState) { - /* check dictMatchState long match */ - U32 const dictMatchIndexL = dictHashLong[dictHL]; - const BYTE* dictMatchL = dictBase + dictMatchIndexL; - assert(dictMatchL < dictEnd); - if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { - mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; - offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); - while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ - goto _match_found; - } } + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + /* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch. + * However expression below complies into conditional move. Since + * match is unlikely and we only *branch* on idxl0 > prefixLowestIndex + * if there is a match, all branches become predictable. */ + { const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]); + + /* check prefix long match */ + if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + /* Same optimization as matchl0 above */ + matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]); - if (matchIndexS > prefixLowestIndex) { /* check prefix short match */ - if (MEM_read32(match) == MEM_read32(ip)) { - goto _search_next_long; + if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) { + goto _search_next_long; } - } else if (dictMode == ZSTD_dictMatchState) { - /* check dictMatchState short match */ - U32 const dictMatchIndexS = dictHashSmall[dictHS]; - match = dictBase + dictMatchIndexS; - matchIndexS = dictMatchIndexS + dictIndexDelta; - if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { - goto _search_next_long; - } } + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; - ip += ((ip-anchor) >> kSearchStrength) + 1; -#if defined(__aarch64__) - PREFETCH_L1(ip+256); -#endif - continue; + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); _search_next_long: - { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); - U32 const matchIndexL3 = hashLong[hl3]; - const BYTE* matchL3 = base + matchIndexL3; - hashLong[hl3] = curr + 1; + /* short match found: let's check for a longer one */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); - /* check prefix long +1 match */ - if (matchIndexL3 > prefixLowestIndex) { - if (MEM_read64(matchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; - ip++; - offset = (U32)(ip-matchL3); - while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ - goto _match_found; - } - } else if (dictMode == ZSTD_dictMatchState) { - /* check dict long +1 match */ - U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; - const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; - assert(dictMatchL3 < dictEnd); - if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; - ip++; - offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); - while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ - goto _match_found; - } } } - - /* if no long +1 match, explore the short match we found */ - if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { - mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; - offset = (U32)(curr - matchIndexS); - while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - } else { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - offset = (U32)(ip - match); - while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + /* check long match at +1 position */ + if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) { + size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; + if (l1len > mLength) { + /* use the long match instead */ + ip = ip1; + mLength = l1len; + offset = (U32)(ip-matchl1); + matchs0 = matchl1; + } } + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */ + /* fall-through */ -_match_found: +_match_found: /* requires ip, offset, mLength */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); _match_stored: /* match found */ @@ -20016,56 +28041,268 @@ _match_stored: } /* check immediate repcode */ - if (dictMode == ZSTD_dictMatchState) { - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState - && repIndex2 < prefixLowestIndex ? - dictBase + repIndex2 - dictIndexDelta : - base + repIndex2; - if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } +} - if (dictMode == ZSTD_noDict) { - while ( (ip <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + + /* if a dictionary is attached, it must be within window range */ + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes); + PREFETCH_AREA(dictHashSmall, chainTableBytes); + } + + /* init */ + ip += (dictAndPrefixLength == 0); + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check repcode */ + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + /* check prefix long match */ + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } else if (dictTagsMatchL) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* short match candidate */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictTagsMatchS) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } else if (dictTagsMatchL3) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; } } } + + /* if no long +1 match, explore the short match we found */ + if (matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } } /* while (ip < ilimit) */ /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); } +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; @@ -20073,19 +28310,19 @@ size_t ZSTD_compressBlock_doubleFast( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); } } size_t ZSTD_compressBlock_doubleFast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { const U32 mls = ms->cParams.minMatch; @@ -20093,19 +28330,21 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); } } -static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls /* template */) { @@ -20135,7 +28374,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ @@ -20156,13 +28395,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( size_t mLength; hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ - if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ - & (repIndex > dictStartIndex)) + if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -20173,7 +28412,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -20198,7 +28437,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -20224,13 +28463,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ - & (repIndex2 > dictStartIndex)) + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) + & (offset_2 <= current2 - dictStartIndex)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -20248,9 +28487,13 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( return (size_t)(iend - anchor); } +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; @@ -20258,19 +28501,21 @@ size_t ZSTD_compressBlock_doubleFast_extDict( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); } } + +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ /**** ended inlining compress/zstd_double_fast.c ****/ /**** start inlining compress/zstd_fast.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20282,8 +28527,46 @@ size_t ZSTD_compressBlock_doubleFast_extDict( /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_fast.h ****/ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; -void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms, const void* const end, ZSTD_dictTableLoadMethod_e dtlm) { @@ -20296,6 +28579,10 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const U32 fastHashFillStep = 3; + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + /* Always insert every fastHashFillStep position into the hash table. * Insert the other positions if their hash entry is empty. */ @@ -20313,171 +28600,405 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } +void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_fast_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + +typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); + +static int +ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* Array of ~random data, should have low probability of matching data. + * Load from here if the index is invalid. + * Used to avoid unpredictable branches. */ + static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; + + /* currentIdx >= lowLimit is a (somewhat) unpredictable branch. + * However expression below compiles into conditional move. + */ + const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); + /* Note: this used to be written as : return test1 && test2; + * Unfortunately, once inlined, these tests become branches, + * in which case it becomes critical that they are executed in the right order (test1 then test2). + * So we have to write these tests in a specific manner to ensure their ordering. + */ + if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; + /* force ordering of these tests, which matters once the function is inlined, as they become branches */ +#if defined(__GNUC__) + __asm__(""); +#endif + return matchIdx >= idxLowLimit; +} + +static int +ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) +{ + /* using a branch instead of a cmov, + * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, + * aka almost all candidates are within range */ + U32 mval; + if (matchIdx >= idxLowLimit) { + mval = MEM_read32(matchAddress); + } else { + mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ + } + + return (MEM_read32(currentPtr) == mval); +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_noDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls) + U32 const mls, int useCmov) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; U32 const hlog = cParams->hashLog; - /* support stepSize of 0 */ - size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */ const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */ - const BYTE* ip0 = istart; - const BYTE* ip1; - const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; - /* init */ + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 matchIdx; /* match idx for ip0 */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); - ip1 = ip0 + 1; { U32 const curr = (U32)(ip0 - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; } - /* Main Search Loop */ -#ifdef __INTEL_COMPILER - /* From intel 'The vector pragma indicates that the loop should be - * vectorized if it is legal to do so'. Can be used together with - * #pragma ivdep (but have opted to exclude that because intel - * warns against using it).*/ - #pragma vector always -#endif - while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */ - size_t mLength; - BYTE const* ip2 = ip0 + 2; - size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls); - U32 const val0 = MEM_read32(ip0); - size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls); - U32 const val1 = MEM_read32(ip1); - U32 const current0 = (U32)(ip0-base); - U32 const current1 = (U32)(ip1-base); - U32 const matchIndex0 = hashTable[h0]; - U32 const matchIndex1 = hashTable[h1]; - BYTE const* repMatch = ip2 - offset_1; - const BYTE* match0 = base + matchIndex0; - const BYTE* match1 = base + matchIndex1; - U32 offcode; + /* start each op */ +_start: /* Requires: ip0 */ -#if defined(__aarch64__) - PREFETCH_L1(ip0+256); -#endif + step = stepSize; + nextStep = ip0 + kStepIncr; - hashTable[h0] = current0; /* update hash table */ - hashTable[h1] = current1; /* update hash table */ + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; - assert(ip0 + 1 == ip1); + if (ip3 >= ilimit) { + goto _cleanup; + } - if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) { - mLength = (ip2[-1] == repMatch[-1]) ? 1 : 0; - ip0 = ip2 - mLength; - match0 = repMatch - mLength; + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + matchIdx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; mLength += 4; - offcode = 0; + + /* Write next hash table entry: it's already calculated. + * This write is known to be safe because ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + goto _match; } - if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) { - /* found a regular match */ + + if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { + /* Write next hash table entry (it's already calculated). + * This write is known to be safe because the ip1 == ip0 + 1, + * so searching will resume after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + goto _offset; } - if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) { - /* found a regular match after one literal */ - ip0 = ip1; - match0 = match1; + + /* lookup ip[1] */ + matchIdx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { + /* Write next hash table entry, since it's already calculated */ + if (step <= 4) { + /* Avoid writing an index if it's >= position where search will resume. + * The minimum possible match has length 4, so search can resume at ip0 + 4. + */ + hashTable[hash1] = (U32)(ip1 - base); + } goto _offset; } - { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; - assert(step >= 2); - ip0 += step; - ip1 += step; - continue; + + /* lookup ip[1] */ + matchIdx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; } -_offset: /* Requires: ip0, match0 */ - /* Compute the offset code */ - offset_2 = offset_1; - offset_1 = (U32)(ip0-match0); - offcode = offset_1 + ZSTD_REP_MOVE; - mLength = 4; - /* Count the backwards match length */ - while (((ip0>anchor) & (match0>prefixStart)) - && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */ + } while (ip3 < ilimit); -_match: /* Requires: ip0, match0, offcode */ - /* Count the forward length */ - mLength += ZSTD_count(ip0+mLength, match0+mLength, iend); - ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); - /* match found */ - ip0 += mLength; - anchor = ip0; +_cleanup: + /* Note that there are probably still a couple positions one could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ - if (ip0 <= ilimit) { - /* Fill Table */ - assert(base+current0+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - - if (offset_2 > 0) { /* offset_2==0 means offset_2 is invalidated */ - while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); - ip0 += rLength; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); - anchor = ip0; - continue; /* faster when present (confirmed on gcc-8) ... (?) */ - } } } - ip1 = ip0 + 1; - } + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + matchIdx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = OFFSET_TO_OFFBASE(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; } +#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \ + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - U32 const mls = ms->cParams.minMatch; + U32 const mml = ms->cParams.minMatch; + /* use cmov when "candidate in range" branch is likely unpredictable */ + int const useCmov = ms->cParams.windowLog < 19; assert(ms->dictMatchState == NULL); - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); - case 5 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); - case 6 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); - case 7 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); + if (useCmov) { + switch(mml) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + /* use a branch instead */ + switch(mml) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } } } FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_fast_dictMatchState_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls) + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -20486,16 +29007,16 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( U32 const stepSize = cParams->targetLength + !(cParams->targetLength); const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ const BYTE* anchor = istart; const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; - const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_MatchState_t* const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; const U32* const dictHashTable = dms->hashTable; const U32 dictStartIndex = dms->window.dictLimit; @@ -20503,127 +29024,183 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dms->window.nextSrc; const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); - const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); - const U32 dictHLog = dictCParams->hashLog; + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; /* if a dictionary is still attached, it necessarily means that * it is within window size. So we just check it. */ const U32 maxDistance = 1U << cParams->windowLog; - const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); assert(endIndex - prefixStartIndex <= maxDistance); (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + (void)hasStep; /* not currently specialized on whether it's accelerated */ + /* ensure there will be no underflow * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes); + } + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); - ip += (dictAndPrefixLength == 0); + ip0 += (dictAndPrefixLength == 0); /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hlog, mls); - U32 const curr = (U32)(ip-base); - U32 const matchIndex = hashTable[h]; - const BYTE* match = base + matchIndex; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (repIndex < prefixStartIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - hashTable[h] = curr; /* update hash table */ + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); - if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); - } else if ( (matchIndex <= prefixStartIndex) ) { - size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); - U32 const dictMatchIndex = dictHashTable[dictHash]; - const BYTE* dictMatch = dictBase + dictMatchIndex; - if (dictMatchIndex <= dictStartIndex || - MEM_read32(dictMatch) != MEM_read32(ip)) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a dict match */ - U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); - mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; - while (((ip>anchor) & (dictMatch>dictStart)) - && (ip[-1] == dictMatch[-1])) { - ip--; dictMatch--; mLength++; + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) { + /* found a regular match of size >= 4 */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; } - } else if (MEM_read32(match) != MEM_read32(ip)) { - /* it's not a match, and we're not going to check the dictionary */ - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a regular match */ - U32 const offset = (U32)(ip-match); - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - while (((ip>anchor) & (match>prefixStart)) - && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ /* match found */ - ip += mLength; - anchor = ip; + assert(mLength); + ip0 += mLength; + anchor = ip0; - if (ip <= ilimit) { + if (ip0 <= ilimit) { /* Fill Table */ assert(base+curr+2 > istart); /* check base overflow */ hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase - dictIndexDelta + repIndex2 : base + repIndex2; - if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; continue; } break; } } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; } +_cleanup: /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); } + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; @@ -20632,30 +29209,31 @@ size_t ZSTD_compressBlock_fast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); } } -static size_t ZSTD_compressBlock_fast_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls) +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; U32 const hlog = cParams->hashLog; /* support stepSize of 0 */ - U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); @@ -20668,107 +29246,263 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ + + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + (void)hasStep; /* not currently specialized on whether it's accelerated */ DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hlog, mls); - const U32 matchIndex = hashTable[h]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - hashTable[h] = curr; /* update hash table */ - DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); - assert(offset_1 <= curr +1); /* check repIndex */ + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } - if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); - ip += rLength; - anchor = ip; - } else { - if ( (matchIndex < dictStartIndex) || - (MEM_read32(match) != MEM_read32(ip)) ) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ } - { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 const offset = curr - matchIndex; - size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = offset; /* update offset history */ - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - ip += mLength; - anchor = ip; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; } } - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } } + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx, idxBase */ + + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0)) + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; } +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); switch(mls) { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); case 5 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); case 6 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); case 7 : - return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); } } /**** ended inlining compress/zstd_fast.c ****/ /**** start inlining compress/zstd_lazy.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20779,14 +29513,23 @@ size_t ZSTD_compressBlock_fast_extDict( /**** skipping file: zstd_compress_internal.h ****/ /**** skipping file: zstd_lazy.h ****/ +/**** skipping file: ../common/bits.h ****/ + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) + +#define kLazySkippingStep 8 /*-************************************* * Binary Tree search ***************************************/ -static void -ZSTD_updateDUBT(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_updateDUBT(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend, U32 mls) { @@ -20829,8 +29572,9 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms, * sort one already inserted but unsorted position * assumption : curr >= btlow == (curr - btmask) * doesn't fail */ -static void -ZSTD_insertDUBT1(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms, U32 curr, const BYTE* inputEnd, U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode) @@ -20862,7 +29606,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, assert(curr >= btLow); assert(ip < iend); /* condition for ZSTD_count */ - while (nbCompares-- && (matchIndex > windowLow)) { + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < curr); @@ -20918,9 +29662,10 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, } -static size_t -ZSTD_DUBT_findBetterDictMatch ( - ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, size_t* offsetPtr, size_t bestLength, @@ -20928,7 +29673,7 @@ ZSTD_DUBT_findBetterDictMatch ( U32 const mls, const ZSTD_dictMode_e dictMode) { - const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_MatchState_t * const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; const U32 * const dictHashTable = dms->hashTable; U32 const hashLog = dmsCParams->hashLog; @@ -20954,7 +29699,7 @@ ZSTD_DUBT_findBetterDictMatch ( (void)dictMode; assert(dictMode == ZSTD_dictMatchState); - while (nbCompares-- && (dictMatchIndex > dictLowLimit)) { + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match = dictBase + dictMatchIndex; @@ -20966,8 +29711,8 @@ ZSTD_DUBT_findBetterDictMatch ( U32 matchIndex = dictMatchIndex + dictIndexDelta; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", - curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + curr - matchIndex, dictMatchIndex, matchIndex); - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); } if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ @@ -20987,7 +29732,7 @@ ZSTD_DUBT_findBetterDictMatch ( } if (bestLength >= MINMATCH) { - U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", curr, (U32)bestLength, (U32)*offsetPtr, mIndex); } @@ -20996,10 +29741,11 @@ ZSTD_DUBT_findBetterDictMatch ( } -static size_t -ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, - size_t* offsetPtr, + size_t* offBasePtr, U32 const mls, const ZSTD_dictMode_e dictMode) { @@ -21078,7 +29824,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, matchIndex = hashTable[h]; hashTable[h] = curr; /* Update Hash Table */ - while (nbCompares-- && (matchIndex > windowLow)) { + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; @@ -21096,8 +29842,8 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ if (dictMode == ZSTD_dictMatchState) { nbCompares = 0; /* in addition to avoiding checking any @@ -21126,19 +29872,20 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, *smallerPtr = *largerPtr = 0; + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ if (dictMode == ZSTD_dictMatchState && nbCompares) { bestLength = ZSTD_DUBT_findBetterDictMatch( ms, ip, iend, - offsetPtr, bestLength, nbCompares, + offBasePtr, bestLength, nbCompares, mls, dictMode); } assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ if (bestLength >= MINMATCH) { - U32 const mIndex = curr - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex; + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); } return bestLength; } @@ -21146,106 +29893,25 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, /** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ -FORCE_INLINE_TEMPLATE size_t -ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, + size_t* offBasePtr, const U32 mls /* template */, const ZSTD_dictMode_e dictMode) { DEBUGLOG(7, "ZSTD_BtFindBestMatch"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateDUBT(ms, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); } - -static size_t -ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); - } -} - - -static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); - } -} - - -static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); - } -} - - - -/* ********************************* -* Hash Chain +/*********************************** +* Dedicated dict search ***********************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] -/* Update chains up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( - ZSTD_matchState_t* ms, - const ZSTD_compressionParameters* const cParams, - const BYTE* ip, U32 const mls) -{ - U32* const hashTable = ms->hashTable; - const U32 hashLog = cParams->hashLog; - U32* const chainTable = ms->chainTable; - const U32 chainMask = (1 << cParams->chainLog) - 1; - const BYTE* const base = ms->window.base; - const U32 target = (U32)(ip - base); - U32 idx = ms->nextToUpdate; - - while(idx < target) { /* catch up */ - size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); - NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; - hashTable[h] = idx; - idx++; - } - - ms->nextToUpdate = target; - return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; -} - -U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); -} - -void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip) { const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); @@ -21253,7 +29919,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B U32* const chainTable = ms->chainTable; U32 const chainSize = 1 << ms->cParams.chainLog; U32 idx = ms->nextToUpdate; - U32 const minChain = chainSize < target ? target - chainSize : idx; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; U32 const cacheSize = bucketSize - 1; U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; @@ -21267,13 +29933,12 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; U32* const tmpHashTable = hashTable; U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); - U32 const tmpChainSize = ((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; - U32 hashIdx; assert(ms->cParams.chainLog <= 24); - assert(ms->cParams.hashLog >= ms->cParams.chainLog); + assert(ms->cParams.hashLog > ms->cParams.chainLog); assert(idx != 0); assert(tmpMinChain <= minChain); @@ -21304,7 +29969,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B if (count == cacheSize) { for (count = 0; count < chainLimit;) { if (i < minChain) { - if (!i || countBeyondMinChain++ > cacheSize) { + if (!i || ++countBeyondMinChain > cacheSize) { /* only allow pulling `cacheSize` number of entries * into the cache or chainTable beyond `minChain`, * to replace the entries pulled out of the @@ -21360,11 +30025,150 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B ms->nextToUpdate = target; } +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_MatchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; +} + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_MatchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls, U32 const lazySkipping) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); +} /* inlining is important to hardwire a hot branch (template emulation) */ FORCE_INLINE_TEMPLATE -size_t ZSTD_HcFindBestMatch_generic ( - ZSTD_matchState_t* ms, +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_HcFindBestMatch( + ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, size_t* offsetPtr, const U32 mls, const ZSTD_dictMode_e dictMode) @@ -21388,7 +30192,7 @@ size_t ZSTD_HcFindBestMatch_generic ( U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; - const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_MatchState_t* const dms = ms->dictMatchState; const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch @@ -21402,14 +30206,15 @@ size_t ZSTD_HcFindBestMatch_generic ( } /* HC4 match finder */ - matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ - if (match[ml] == ip[ml]) /* potentially better */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ currentMl = ZSTD_count(ip, match, iLimit); } else { const BYTE* const match = dictBase + matchIndex; @@ -21421,7 +30226,7 @@ size_t ZSTD_HcFindBestMatch_generic ( /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } @@ -21429,91 +30234,10 @@ size_t ZSTD_HcFindBestMatch_generic ( matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); } + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ if (dictMode == ZSTD_dedicatedDictSearch) { - const U32 ddsLowestIndex = dms->window.dictLimit; - const BYTE* const ddsBase = dms->window.base; - const BYTE* const ddsEnd = dms->window.nextSrc; - const U32 ddsSize = (U32)(ddsEnd - ddsBase); - const U32 ddsIndexDelta = dictLimit - ddsSize; - const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); - const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; - U32 ddsAttempt; - - for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { - PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); - } - - { - U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; - U32 const chainIndex = chainPackedPointer >> 8; - - PREFETCH_L1(&dms->chainTable[chainIndex]); - } - - for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { - size_t currentMl=0; - const BYTE* match; - matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; - match = ddsBase + matchIndex; - - if (!matchIndex) { - return ml; - } - - /* guaranteed by table construction */ - (void)ddsLowestIndex; - assert(matchIndex >= ddsLowestIndex); - assert(match+4 <= ddsEnd); - if (MEM_read32(match) == MEM_read32(ip)) { - /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; - } - - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) { - /* best possible, avoids read overflow on next attempt */ - return ml; - } - } - } - - { - U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; - U32 chainIndex = chainPackedPointer >> 8; - U32 const chainLength = chainPackedPointer & 0xFF; - U32 const chainAttempts = nbAttempts - ddsAttempt; - U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; - U32 chainAttempt; - - for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { - PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); - } - - for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { - size_t currentMl=0; - const BYTE* match; - matchIndex = dms->chainTable[chainIndex]; - match = ddsBase + matchIndex; - - /* guaranteed by table construction */ - assert(matchIndex >= ddsLowestIndex); - assert(match+4 <= ddsEnd); - if (MEM_read32(match) == MEM_read32(ip)) { - /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; - } - - /* save best solution */ - if (currentMl > ml) { - ml = currentMl; - *offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; - if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ - } - } - } + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); } else if (dictMode == ZSTD_dictMatchState) { const U32* const dmsChainTable = dms->chainTable; const U32 dmsChainSize = (1 << dms->cParams.chainLog); @@ -21537,7 +30261,8 @@ size_t ZSTD_HcFindBestMatch_generic ( /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } @@ -21550,79 +30275,749 @@ size_t ZSTD_HcFindBestMatch_generic ( return ml; } +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); +} + +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; +} + +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; +} + +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ +} + +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_fillHashCache(ZSTD_MatchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) { - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + U32 const* const hashTable = ms->hashTable; + BYTE const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); + + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); +} + +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + BYTE const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls, + U64 const hashSalt) +{ + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; } } - -static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) { - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = tagTable + relRow; + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; } } - -static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) { - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} + +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); + + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} + +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; + } + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; +} + +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) +{ + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; i> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, headGrouped); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, headGrouped); + } else { + return ZSTD_rotateRight_U64((U64)matches, headGrouped); + } + } +#endif +} + +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index. + * - The hash is salted by a value that changes on every context reset, so when the same table is used + * we will avoid collisions that would otherwise slow us down by introducing phantom matches. + * - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can + * be considered as a circular buffer with a "head" index that resides in the tagTable (overall 16 or 32 bytes + * per row). + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte tag calculated for the position and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + * - Insert the tag into the equivalent row and position in the tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_RowFindBestMatch( + ZSTD_MatchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + U32 hash; + + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); + } + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; + } + + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + BYTE* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + + { /* Get the hash for ip, compute the appropriate row */ + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); + + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + } + + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } + } + } + } + return ml; } -FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* const iLimit, - size_t* offsetPtr) -{ - switch(ms->cParams.minMatch) - { - default : /* includes case 3 */ - case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); - case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); - case 7 : - case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); - } -} +/** + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_MatchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog + +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ + ZSTD_MatchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) + +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ + } + +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; + +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; + +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_MatchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) + } + ZSTD_UNREACHABLE; + return 0; +} /* ******************************* * Common parser - lazy strategy *********************************/ -typedef enum { search_hashChain, search_binaryTree } searchMethod_e; -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_lazy_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_lazy_generic( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const searchMethod_e searchMethod, const U32 depth, @@ -21632,47 +31027,20 @@ ZSTD_compressBlock_lazy_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; const BYTE* const base = ms->window.base; const U32 prefixLowestIndex = ms->window.dictLimit; const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - - /** - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ - const searchMax_f searchFuncs[4][2] = { - { - ZSTD_HcFindBestMatch_selectMLS, - ZSTD_BtFindBestMatch_selectMLS - }, - { - NULL, - NULL - }, - { - ZSTD_HcFindBestMatch_dictMatchState_selectMLS, - ZSTD_BtFindBestMatch_dictMatchState_selectMLS - }, - { - ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, - NULL - } - }; - - searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; const int isDMS = dictMode == ZSTD_dictMatchState; const int isDDS = dictMode == ZSTD_dedicatedDictSearch; const int isDxS = isDMS || isDDS; - const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_MatchState_t* const dms = ms->dictMatchState; const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; const BYTE* const dictBase = isDxS ? dms->window.base : NULL; const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; @@ -21682,18 +31050,14 @@ ZSTD_compressBlock_lazy_generic( 0; const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); - assert(searchMax != NULL); - - DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode); - - /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { U32 const curr = (U32)(ip - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; - if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; } if (isDxS) { /* dictMatchState repCode checks don't currently handle repCode == 0 @@ -21702,6 +31066,13 @@ ZSTD_compressBlock_lazy_generic( assert(offset_2 <= dictAndPrefixLength); } + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + /* Match Loop */ #if defined(__GNUC__) && defined(__x86_64__) /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the @@ -21711,8 +31082,9 @@ ZSTD_compressBlock_lazy_generic( #endif while (ip < ilimit) { size_t matchLength=0; - size_t offset=0; + size_t offBase = REPCODE1_TO_OFFBASE; const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); /* check repCode */ if (isDxS) { @@ -21721,7 +31093,7 @@ ZSTD_compressBlock_lazy_generic( && repIndex < prefixLowestIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; @@ -21735,116 +31107,133 @@ ZSTD_compressBlock_lazy_generic( } /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); if (ml2 > matchLength) - matchLength = ml2, start = ip, offset=offsetFound; + matchLength = ml2, start = ip, offBase = offbaseFound; } if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; continue; } /* let's try to find a better solution */ if (depth>=1) while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; /* search a better one */ } } /* let's find an even better one */ if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; - if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip)) ) { const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offset = 0, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; } } } break; /* nothing found : store previous solution */ } /* NOTE: - * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. - * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which - * overflows the pointer, which is undefined behavior. + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. */ /* catch up */ - if (offset) { + if (OFFBASE_IS_OFFSET(offBase)) { if (dictMode == ZSTD_noDict) { - while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > prefixLowest)) - && (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */ + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ { start--; matchLength++; } } if (isDxS) { - U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ } - offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); } /* store sequence */ _storeSequence: - { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); anchor = ip = start + matchLength; } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } /* check immediate repcode */ if (isDxS) { @@ -21854,12 +31243,12 @@ _storeSequence: const BYTE* repMatch = repIndex < prefixLowestIndex ? dictBase - dictIndexDelta + repIndex : base + repIndex; - if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) && (MEM_read32(repMatch) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; @@ -21873,104 +31262,183 @@ _storeSequence: && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ } } } - /* Save reps for next block */ - rep[0] = offset_1 ? offset_1 : savedOffset; - rep[1] = offset_2 ? offset_2 : savedOffset; + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); } +#endif /* build exclusions */ -size_t ZSTD_compressBlock_btlazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); -} - -size_t ZSTD_compressBlock_lazy2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); -} - -size_t ZSTD_compressBlock_lazy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); -} - +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_greedy( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); } -size_t ZSTD_compressBlock_btlazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); -} - -size_t ZSTD_compressBlock_lazy2_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); -} - -size_t ZSTD_compressBlock_lazy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); -} - size_t ZSTD_compressBlock_greedy_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); } - -size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); -} - -size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); -} - size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); } +size_t ZSTD_compressBlock_greedy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} +#endif + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_compressBlock_lazy_extDict_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const searchMethod_e searchMethod, const U32 depth) @@ -21979,7 +31447,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; const BYTE* const base = ms->window.base; const U32 dictLimit = ms->window.dictLimit; const BYTE* const prefixStart = base + dictLimit; @@ -21987,18 +31455,21 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const dictStart = dictBase + ms->window.lowLimit; const U32 windowLog = ms->cParams.windowLog; - - typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); U32 offset_1 = rep[0], offset_2 = rep[1]; - DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic"); + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; /* init */ ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } /* Match Loop */ #if defined(__GNUC__) && defined(__x86_64__) @@ -22009,7 +31480,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( #endif while (ip < ilimit) { size_t matchLength=0; - size_t offset=0; + size_t offBase = REPCODE1_TO_OFFBASE; const BYTE* start=ip+1; U32 curr = (U32)(ip-base); @@ -22018,7 +31489,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const U32 repIndex = (U32)(curr+1 - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ if (MEM_read32(ip+1) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -22027,14 +31499,23 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( } } /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); if (ml2 > matchLength) - matchLength = ml2, start = ip, offset=offsetFound; + matchLength = ml2, start = ip, offBase = ofbCandidate; } - if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; continue; } @@ -22044,29 +31525,30 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ip ++; curr++; /* check repCode */ - if (offset) { + if (offBase) { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; } } /* search match, depth 1 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; /* search a better one */ } } @@ -22075,49 +31557,57 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ip ++; curr++; /* check repCode */ - if (offset) { + if (offBase) { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; int const gain2 = (int)(repLength * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offset = 0, start = ip; + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; } } /* search match, depth 2 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offset = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; } } } break; /* nothing found : store previous solution */ } /* catch up */ - if (offset) { - U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); } /* store sequence */ _storeSequence: - { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); anchor = ip = start + matchLength; } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } /* check immediate repcode */ while (ip <= ilimit) { @@ -22126,13 +31616,14 @@ _storeSequence: const U32 repIndex = repCurrent - offset_2; const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ + if ( (ZSTD_index_overlap_check(dictLimit, repIndex)) + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; - offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -22147,42 +31638,72 @@ _storeSequence: /* Return the last literals size */ return (size_t)(iend - anchor); } +#endif /* build exclusions */ - +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_greedy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); } +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_lazy_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); } +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_lazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); } +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btlazy2_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); } +#endif /**** ended inlining compress/zstd_lazy.c ****/ /**** start inlining compress/zstd_ldm.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -22199,7 +31720,7 @@ size_t ZSTD_compressBlock_btlazy2_extDict( /**** skipping file: zstd_double_fast.h ****/ /**** start inlining zstd_ldm_geartab.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -22211,7 +31732,10 @@ size_t ZSTD_compressBlock_btlazy2_extDict( #ifndef ZSTD_LDM_GEARTAB_H #define ZSTD_LDM_GEARTAB_H -static U64 ZSTD_ldm_gearTab[256] = { +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/mem.h ****/ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, @@ -22303,7 +31827,7 @@ static U64 ZSTD_ldm_gearTab[256] = { #endif /* ZSTD_LDM_GEARTAB_H */ /**** ended inlining zstd_ldm_geartab.h ****/ -#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_BUCKET_SIZE_LOG 4 #define LDM_MIN_MATCH_LENGTH 64 #define LDM_HASH_RLOG 7 @@ -22344,6 +31868,33 @@ static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* } } +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) +{ + U64 hash = state->rolling; + size_t n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE +} + /** ZSTD_ldm_gear_feed(): * * Registers in the splits array all the split points found in the first @@ -22393,21 +31944,35 @@ done: } void ZSTD_ldm_adjustParameters(ldmParams_t* params, - ZSTD_compressionParameters const* cParams) + const ZSTD_compressionParameters* cParams) { params->windowLog = cParams->windowLog; ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); - if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; - if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; - if (params->hashLog == 0) { - params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); - assert(params->hashLog <= ZSTD_HASHLOG_MAX); - } if (params->hashRateLog == 0) { - params->hashRateLog = params->windowLog < params->hashLog - ? 0 - : params->windowLog - params->hashLog; + if (params->hashLog > 0) { + /* if params->hashLog is set, derive hashRateLog from it */ + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + if (params->windowLog > params->hashLog) { + params->hashRateLog = params->windowLog - params->hashLog; + } + } else { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + /* mapping from [fast, rate7] to [btultra2, rate4] */ + params->hashRateLog = 7 - (cParams->strategy/3); + } + } + if (params->hashLog == 0) { + params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX); + } + if (params->minMatchLength == 0) { + params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (cParams->strategy >= ZSTD_btultra) + params->minMatchLength /= 2; + } + if (params->bucketSizeLog==0) { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX); } params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } @@ -22419,33 +31984,33 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params) size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); - return params.enableLdm ? totalSize : 0; + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; } size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) { - return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0; + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; } /** ZSTD_ldm_getBucket() : * Returns a pointer to the start of the bucket associated with hash. */ static ldmEntry_t* ZSTD_ldm_getBucket( - ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) + const ldmState_t* ldmState, size_t hash, U32 const bucketSizeLog) { - return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); + return ldmState->hashTable + (hash << bucketSizeLog); } /** ZSTD_ldm_insertEntry() : * Insert the entry with corresponding hash into the hash table */ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, size_t const hash, const ldmEntry_t entry, - ldmParams_t const ldmParams) + U32 const bucketSizeLog) { BYTE* const pOffset = ldmState->bucketOffsets + hash; unsigned const offset = *pOffset; - *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; - *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + *(ZSTD_ldm_getBucket(ldmState, hash, bucketSizeLog) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << bucketSizeLog) - 1)); } @@ -22494,7 +32059,7 @@ static size_t ZSTD_ldm_countBackwardsMatch_2segments( * * The tables for the other strategies are filled within their * block compressors. */ -static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, +static size_t ZSTD_ldm_fillFastTables(ZSTD_MatchState_t* ms, void const* end) { const BYTE* const iend = (const BYTE*)end; @@ -22502,11 +32067,15 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, switch(ms->cParams.strategy) { case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif break; case ZSTD_greedy: @@ -22529,7 +32098,8 @@ void ZSTD_ldm_fillHashTable( const BYTE* iend, ldmParams_t const* params) { U32 const minMatchLength = params->minMatchLength; - U32 const hBits = params->hashLog - params->bucketSizeLog; + U32 const bucketSizeLog = params->bucketSizeLog; + U32 const hBits = params->hashLog - bucketSizeLog; BYTE const* const base = ldmState->window.base; BYTE const* const istart = ip; ldmRollingHashState_t hashState; @@ -22544,7 +32114,7 @@ void ZSTD_ldm_fillHashTable( unsigned n; numSplits = 0; - hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + hashed = ZSTD_ldm_gear_feed(&hashState, ip, (size_t)(iend - ip), splits, &numSplits); for (n = 0; n < numSplits; n++) { if (ip + splits[n] >= istart + minMatchLength) { @@ -22555,7 +32125,7 @@ void ZSTD_ldm_fillHashTable( entry.offset = (U32)(split - base); entry.checksum = (U32)(xxhash >> 32); - ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + ZSTD_ldm_insertEntry(ldmState, hash, entry, params->bucketSizeLog); } } @@ -22569,7 +32139,7 @@ void ZSTD_ldm_fillHashTable( * Sets cctx->nextToUpdate to a position corresponding closer to anchor * if it is far way * (after a long match, only update tables a limited amount). */ -static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor) { U32 const curr = (U32)(anchor - ms->window.base); if (curr > ms->nextToUpdate + 1024) { @@ -22578,8 +32148,10 @@ static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) } } -static size_t ZSTD_ldm_generateSequences_internal( - ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, RawSeqStore_t* rawSeqStore, ldmParams_t const* params, void const* src, size_t srcSize) { /* LDM parameters */ @@ -22614,16 +32186,8 @@ static size_t ZSTD_ldm_generateSequences_internal( /* Initialize the rolling hash state with the first minMatchLength bytes */ ZSTD_ldm_gear_init(&hashState, params); - { - size_t n = 0; - - while (n < minMatchLength) { - numSplits = 0; - n += ZSTD_ldm_gear_feed(&hashState, ip + n, minMatchLength - n, - splits, &numSplits); - } - ip += minMatchLength; - } + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; while (ip < ilimit) { size_t hashed; @@ -22641,13 +32205,14 @@ static size_t ZSTD_ldm_generateSequences_internal( candidates[n].split = split; candidates[n].hash = hash; candidates[n].checksum = (U32)(xxhash >> 32); - candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, params->bucketSizeLog); PREFETCH_L1(candidates[n].bucket); } for (n = 0; n < numSplits; n++) { size_t forwardMatchLength = 0, backwardMatchLength = 0, bestMatchLength = 0, mLength; + U32 offset; BYTE const* const split = candidates[n].split; U32 const checksum = candidates[n].checksum; U32 const hash = candidates[n].hash; @@ -22663,7 +32228,7 @@ static size_t ZSTD_ldm_generateSequences_internal( * the previous one, we merely register it in the hash table and * move on */ if (split < anchor) { - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); continue; } @@ -22710,14 +32275,14 @@ static size_t ZSTD_ldm_generateSequences_internal( /* No match found -- insert an entry into the hash table * and process the next candidate match */ if (bestEntry == NULL) { - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); continue; } /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; mLength = forwardMatchLength + backwardMatchLength; { - U32 const offset = (U32)(split - base) - bestEntry->offset; rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; /* Out of sequence storage */ @@ -22731,9 +32296,24 @@ static size_t ZSTD_ldm_generateSequences_internal( /* Insert the current entry into the hash table --- it must be * done after the previous block to avoid clobbering bestEntry */ - ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog); anchor = split + forwardMatchLength; + + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. + */ + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } } ip += hashed; @@ -22755,7 +32335,7 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, } size_t ZSTD_ldm_generateSequences( - ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmState_t* ldmState, RawSeqStore_t* sequences, ldmParams_t const* params, void const* src, size_t srcSize) { U32 const maxDist = 1U << params->windowLog; @@ -22787,7 +32367,7 @@ size_t ZSTD_ldm_generateSequences( assert(chunkStart < iend); /* 1. Perform overflow correction if necessary. */ - if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { U32 const ldmHSize = 1U << params->hashLog; U32 const correction = ZSTD_window_correctOverflow( &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); @@ -22801,7 +32381,7 @@ size_t ZSTD_ldm_generateSequences( * the window through early invalidation. * TODO: * Test the chunk size. * * Try invalidation after the sequence generation and test the - * the offset against maxDist directly. + * offset against maxDist directly. * * NOTE: Because of dictionaries + sequence splitting we MUST make sure * that any offset used is valid at the END of the sequence, since it may @@ -22831,7 +32411,9 @@ size_t ZSTD_ldm_generateSequences( return 0; } -void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { +void +ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; if (srcSize <= seq->litLength) { @@ -22866,7 +32448,7 @@ void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 cons * Returns the current sequence to handle, or if the rest of the block should * be literals, it returns a sequence with offset == 0. */ -static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, +static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore, U32 const remaining, U32 const minMatch) { rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; @@ -22890,7 +32472,7 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, return sequence; } -void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { +void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) { U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); while (currPos && rawSeqStore->pos < rawSeqStore->size) { rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; @@ -22907,14 +32489,15 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { } } -size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore, + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_ParamSwitch_e useRowMatchFinder, void const* src, size_t srcSize) { const ZSTD_compressionParameters* const cParams = &ms->cParams; unsigned const minMatch = cParams->minMatch; - ZSTD_blockCompressor const blockCompressor = - ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); + ZSTD_BlockCompressor_f const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); /* Input bounds */ BYTE const* const istart = (BYTE const*)src; BYTE const* const iend = istart + srcSize; @@ -22938,7 +32521,6 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, /* maybeSplitSequence updates rawSeqStore->pos */ rawSeq const sequence = maybeSplitSequence(rawSeqStore, (U32)(iend - ip), minMatch); - int i; /* End signal */ if (sequence.offset == 0) break; @@ -22951,6 +32533,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, /* Run the block compressor */ DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); { + int i; size_t const newLitLength = blockCompressor(ms, seqStore, rep, ip, sequence.litLength); ip += sequence.litLength; @@ -22960,8 +32543,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, rep[0] = sequence.offset; /* Store the sequence */ ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, - sequence.offset + ZSTD_REP_MOVE, - sequence.matchLength - MINMATCH); + OFFSET_TO_OFFBASE(sequence.offset), + sequence.matchLength); ip += sequence.matchLength; } } @@ -22974,7 +32557,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, /**** ended inlining compress/zstd_ldm.c ****/ /**** start inlining compress/zstd_opt.c ****/ /* - * Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -22987,42 +32570,52 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, /**** skipping file: hist.h ****/ /**** skipping file: zstd_opt.h ****/ +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ -#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ #define ZSTD_MAX_PRICE (1<<30) -#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ /*-************************************* * Price functions for optimal parser ***************************************/ -#if 0 /* approximation at bit level */ +#if 0 /* approximation at bit level (for tests) */ # define BITCOST_ACCURACY 0 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) -#elif 0 /* fractional bit accuracy */ +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) #else /* opt==approx, ultra==accurate */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) #endif +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ MEM_STATIC U32 ZSTD_bitWeight(U32 stat) { return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); } +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) { U32 const stat = rawStat + 1; U32 const hb = ZSTD_highbit32(stat); U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; U32 const weight = BWeight + FWeight; assert(hb + BITCOST_ACCURACY < 31); @@ -23033,7 +32626,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) /* debugging function, * @return price in bytes as fractional value * for debug messages only */ -MEM_STATIC double ZSTD_fCost(U32 price) +MEM_STATIC double ZSTD_fCost(int price) { return (double)price / (BITCOST_MULTIPLIER*8); } @@ -23041,7 +32634,7 @@ MEM_STATIC double ZSTD_fCost(U32 price) static int ZSTD_compressedLiterals(optState_t const* const optPtr) { - return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; + return optPtr->literalCompressionMode != ZSTD_ps_disable; } static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) @@ -23054,25 +32647,52 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) } -/* ZSTD_downscaleStat() : - * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) - * return the resulting sum of elements */ -static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n 0 && ZSTD_FREQ_DIV+malus < 31); + DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", + (unsigned)lastEltIndex+1, (unsigned)shift ); + assert(shift < 30); for (s=0; s> (ZSTD_FREQ_DIV+malus)); - sum += table[s]; + unsigned const base = base1 ? 1 : (table[s]>0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; } return sum; } +/* ZSTD_scaleStats() : + * reduce all elt frequencies in table if sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); +} + /* ZSTD_rescaleFreqs() : * if first block (detected by optPtr->litLengthSum == 0) : init statistics * take hints from dictionary if there is one - * or init from zero, using src for literals stats, or flat 1 for match symbols + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols * otherwise downscale existing stats, to be used as seed for next block. */ static void @@ -23084,24 +32704,28 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); optPtr->priceType = zop_dynamic; - if (optPtr->litLengthSum == 0) { /* first block : init */ - if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ - DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ + + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); optPtr->priceType = zop_predef; } assert(optPtr->symbolCosts != NULL); if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { - /* huffman table presumed generated by dictionary */ + + /* huffman stats covering the full value set : table presumed generated by dictionary */ optPtr->priceType = zop_dynamic; if (compressedLiterals) { + /* generate literals statistics from huffman table */ unsigned lit; assert(optPtr->litFreq != NULL); optPtr->litSum = 0; for (lit=0; lit<=MaxLit; lit++) { U32 const scaleLog = 11; /* scale to 2K */ - U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); assert(bitCost <= scaleLog); optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; optPtr->litSum += optPtr->litFreq[lit]; @@ -23143,20 +32767,26 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->offCodeSum += optPtr->offCodeFreq[of]; } } - } else { /* not a dictionary */ + } else { /* first block, no dictionary */ assert(optPtr->litFreq != NULL); if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); } - { unsigned ll; - for (ll=0; ll<=MaxLL; ll++) - optPtr->litLengthFreq[ll] = 1; + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); } - optPtr->litLengthSum = MaxLL+1; { unsigned ml; for (ml=0; ml<=MaxML; ml++) @@ -23164,21 +32794,25 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } optPtr->matchLengthSum = MaxML+1; - { unsigned of; - for (of=0; of<=MaxOff; of++) - optPtr->offCodeFreq[of] = 1; + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); } - optPtr->offCodeSum = MaxOff+1; } - } else { /* new block : re-use previous statistics, scaled down */ + } else { /* new block : scale down accumulated statistics */ if (compressedLiterals) - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); - optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); } ZSTD_setBasePrices(optPtr, optLevel); @@ -23191,6 +32825,7 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, const optState_t* const optPtr, int optLevel) { + DEBUGLOG(8, "ZSTD_rawLiteralsCost (%u literals)", litLength); if (litLength == 0) return 0; if (!ZSTD_compressedLiterals(optPtr)) @@ -23200,11 +32835,14 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ /* dynamic statistics */ - { U32 price = litLength * optPtr->litSumBasePrice; + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); for (u=0; u < litLength; u++) { - assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ - price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; } return price; } @@ -23214,7 +32852,17 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, * cost of literalLength symbol */ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) { - if (optPtr->priceType == zop_predef) return WEIGHT(litLength, optLevel); + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); /* dynamic statistics */ { U32 const llCode = ZSTD_LLcode(litLength); @@ -23225,22 +32873,25 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP } /* ZSTD_getMatchPrice() : - * Provides the cost of the match part (offset + matchLength) of a sequence + * Provides the cost of the match part (offset + matchLength) of a sequence. * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. - * optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ FORCE_INLINE_TEMPLATE U32 -ZSTD_getMatchPrice(U32 const offset, +ZSTD_getMatchPrice(U32 const offBase, U32 const matchLength, const optState_t* const optPtr, int const optLevel) { U32 price; - U32 const offCode = ZSTD_highbit32(offset+1); + U32 const offCode = ZSTD_highbit32(offBase); U32 const mlBase = matchLength - MINMATCH; assert(matchLength >= MINMATCH); - if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ - return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ /* dynamic statistics */ price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); @@ -23259,10 +32910,10 @@ ZSTD_getMatchPrice(U32 const offset, } /* ZSTD_updateStats() : - * assumption : literals + litLengtn <= iend */ + * assumption : literals + litLength <= iend */ static void ZSTD_updateStats(optState_t* const optPtr, U32 litLength, const BYTE* literals, - U32 offsetCode, U32 matchLength) + U32 offBase, U32 matchLength) { /* literals */ if (ZSTD_compressedLiterals(optPtr)) { @@ -23278,8 +32929,8 @@ static void ZSTD_updateStats(optState_t* const optPtr, optPtr->litLengthSum++; } - /* match offset code (0-2=>repCode; 3+=>offset+2) */ - { U32 const offCode = ZSTD_highbit32(offsetCode+1); + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); assert(offCode <= MaxOff); optPtr->offCodeFreq[offCode]++; optPtr->offCodeSum++; @@ -23313,9 +32964,11 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip) +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) { U32* const hashTable3 = ms->hashTable3; U32 const hashLog3 = ms->hashLog3; @@ -23339,11 +32992,15 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, * Binary Tree search ***************************************/ /** ZSTD_insertBt1() : add one or multiple positions to tree. - * ip : assumed <= iend-8 . + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position * @return : nb of positions added */ -static U32 ZSTD_insertBt1( - ZSTD_matchState_t* ms, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertBt1( + const ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, + U32 const target, U32 const mls, const int extDict) { const ZSTD_compressionParameters* const cParams = &ms->cParams; @@ -23366,7 +33023,10 @@ static U32 ZSTD_insertBt1( U32* smallerPtr = bt + 2*(curr&btMask); U32* largerPtr = smallerPtr + 1; U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = ms->window.lowLimit; + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); U32 matchEndIdx = curr+8+1; size_t bestLength = 8; U32 nbCompares = 1U << cParams->searchLog; @@ -23379,11 +33039,12 @@ static U32 ZSTD_insertBt1( DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + assert(curr <= target); assert(ip <= iend-8); /* required for h calculation */ hashTable[h] = curr; /* Update Hash Table */ assert(windowLow > 0); - while (nbCompares-- && (matchIndex >= windowLow)) { + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < curr); @@ -23455,19 +33116,20 @@ static U32 ZSTD_insertBt1( } FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_updateTree_internal( - ZSTD_matchState_t* ms, + ZSTD_MatchState_t* ms, const BYTE* const ip, const BYTE* const iend, const U32 mls, const ZSTD_dictMode_e dictMode) { const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); U32 idx = ms->nextToUpdate; - DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); while(idx < target) { - U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); assert(idx < (U32)(idx + forward)); idx += forward; } @@ -23476,20 +33138,23 @@ void ZSTD_updateTree_internal( ms->nextToUpdate = target; } -void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { +void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) { ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); } FORCE_INLINE_TEMPLATE -U32 ZSTD_insertBtAndGetAllMatches ( - ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ - ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - const U32 lengthToBeat, - U32 const mls /* template */) +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); @@ -23518,7 +33183,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 mnum = 0; U32 nbCompares = 1U << cParams->searchLog; - const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; const ZSTD_compressionParameters* const dmsCParams = dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; @@ -23557,13 +33222,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( assert(curr >= windowLow); if ( dictMode == ZSTD_extDict && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ - & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; } if (dictMode == ZSTD_dictMatchState && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ - & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; } } @@ -23572,7 +33237,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", repCode, ll0, repOffset, repLen); bestLength = repLen; - matches[mnum].off = repCode - ll0; + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ matches[mnum].len = (U32)repLen; mnum++; if ( (repLen > sufficient_len) @@ -23601,7 +33266,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( bestLength = mlen; assert(curr > matchIndex3); assert(mnum==0); /* no prior solution */ - matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE; + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); matches[0].len = (U32)mlen; mnum = 1; if ( (mlen > sufficient_len) | @@ -23610,11 +33275,11 @@ U32 ZSTD_insertBtAndGetAllMatches ( return 1; } } } /* no dictMatchState lookup: dicts don't have a populated HC3 table */ - } + } /* if (mls == 3) */ hashTable[h] = curr; /* Update Hash Table */ - while (nbCompares-- && (matchIndex >= matchLow)) { + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { U32* const nextPtr = bt + 2*(matchIndex & btMask); const BYTE* match; size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ @@ -23634,21 +33299,20 @@ U32 ZSTD_insertBtAndGetAllMatches ( } if (matchLength > bestLength) { - DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); assert(matchEndIdx > matchIndex); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ break; /* drop, to preserve bt consistency (miss a little bit of compression) */ - } - } + } } if (match[matchLength] < ip[matchLength]) { /* match smaller than current */ @@ -23667,12 +33331,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( *smallerPtr = *largerPtr = 0; + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ if (dictMode == ZSTD_dictMatchState && nbCompares) { size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); U32 dictMatchIndex = dms->hashTable[dmsH]; const U32* const dmsBt = dms->chainTable; commonLengthSmaller = commonLengthLarger = 0; - while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match = dmsBase + dictMatchIndex; @@ -23682,19 +33347,18 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { matchIndex = dictMatchIndex + dmsIndexDelta; - DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - } + } } if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ if (match[matchLength] < ip[matchLength]) { @@ -23704,39 +33368,93 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* match is larger than current */ commonLengthLarger = matchLength; dictMatchIndex = nextPtr[0]; - } - } - } + } } } /* if (dictMode == ZSTD_dictMatchState) */ assert(matchEndIdx > curr+8); ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ return mnum; } +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_MatchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); -FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( - ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ - ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, - U32 const lengthToBeat) +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_MatchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) { - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32 const matchLengthSearch = cParams->minMatch; - DEBUGLOG(8, "ZSTD_BtGetAllMatches"); - if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); - switch(matchLengthSearch) - { - case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); - default : - case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); - case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_MatchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; } /************************* @@ -23745,16 +33463,18 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( /* Struct containing info needed to make decision about ldm inclusion */ typedef struct { - rawSeqStore_t seqStore; /* External match candidates store for this block */ - U32 startPosInBlock; /* Start position of the current match candidate */ - U32 endPosInBlock; /* End position of the current match candidate */ - U32 offset; /* Offset of the match candidate */ + RawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ } ZSTD_optLdm_t; /* ZSTD_optLdm_skipRawSeqStoreBytes(): - * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. */ -static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { +static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) +{ U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); while (currPos && rawSeqStore->pos < rawSeqStore->size) { rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; @@ -23775,8 +33495,10 @@ static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t * Calculates the beginning and end of the next match in the current block. * Updates 'pos' and 'posInSequence' of the ldmSeqStore. */ -static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, - U32 blockBytesRemaining) { +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) +{ rawSeq currSeq; U32 currBlockEndPos; U32 literalsBytesRemaining; @@ -23788,8 +33510,8 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu optLdm->endPosInBlock = UINT_MAX; return; } - /* Calculate appropriate bytes left in matchLength and litLength after adjusting - based on ldmSeqStore->posInSequence */ + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); currBlockEndPos = currPosInBlock + blockBytesRemaining; @@ -23808,7 +33530,7 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu return; } - /* Matches may be < MINMATCH by this process. In that case, we will reject them + /* Matches may be < minMatch by this process. In that case, we will reject them when we are deciding whether or not to add the ldm */ optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; @@ -23825,28 +33547,31 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu } /* ZSTD_optLdm_maybeAddMatch(): - * Adds a match if it's long enough, based on it's 'matchStartPosInBlock' - * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. */ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, - ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { - U32 posDiff = currPosInBlock - optLdm->startPosInBlock; - /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ - U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; - U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 minMatch) +{ + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; /* Ensure that current block position is not outside of the match */ if (currPosInBlock < optLdm->startPosInBlock || currPosInBlock >= optLdm->endPosInBlock - || candidateMatchLength < MINMATCH) { + || candidateMatchLength < minMatch) { return; } if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { - DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", - candidateOffCode, candidateMatchLength, currPosInBlock); + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; + matches[*nbMatches].off = candidateOffBase; (*nbMatches)++; } } @@ -23854,8 +33579,12 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, /* ZSTD_optLdm_processMatchCandidate(): * Wrapper function to update ldm seq store and call ldm functions as necessary. */ -static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches, - U32 currPosInBlock, U32 remainingBytes) { +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes, + U32 minMatch) +{ if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { return; } @@ -23866,24 +33595,19 @@ static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_ * at the end of a match from the ldm seq store, and will often be some bytes * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" */ - U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock; + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); } ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); } - ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch); } + /*-******************************* * Optimal parser *********************************/ - -static U32 ZSTD_totalLen(ZSTD_optimal_t sol) -{ - return sol.litlen + sol.mlen; -} - #if 0 /* debug */ static void @@ -23901,9 +33625,15 @@ listStats(const U32* table, int lastEltID) #endif -FORCE_INLINE_TEMPLATE size_t -ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, - seqStore_t* seqStore, +#define LIT_PRICE(_p) (int)ZSTD_rawLiteralsCost(_p, 1, optStatePtr, optLevel) +#define LL_PRICE(_l) (int)ZSTD_litLengthPrice(_l, optStatePtr, optLevel) +#define LL_INCPRICE(_l) (LL_PRICE(_l) - LL_PRICE(_l-1)) + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t +ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms, + SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const int optLevel, @@ -23919,15 +33649,19 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, const BYTE* const prefixStart = base + ms->window.dictLimit; const ZSTD_compressionParameters* const cParams = &ms->cParams; + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; U32 nextToUpdate3 = ms->nextToUpdate; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; - ZSTD_optimal_t lastSequence; + ZSTD_optimal_t lastStretch; ZSTD_optLdm_t optLdm; + ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t)); + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); @@ -23946,104 +33680,142 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(ip-istart), (U32)(iend - ip)); - if (!nbMatches) { ip++; continue; } + (U32)(ip-istart), (U32)(iend-ip), + minMatch); + if (!nbMatches) { + DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart)); + ip++; + continue; + } + + /* Match found: let's store this solution, and eventually find more candidates. + * During this forward pass, @opt is used to store stretches, + * defined as "a match followed by N literals". + * Note how this is different from a Sequence, which is "N literals followed by a match". + * Storing stretches allows us to store different match predecessors + * for each literal position part of a literals run. */ /* initialize opt[0] */ - { U32 i ; for (i=0; i immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; - U32 const maxOffset = matches[nbMatches-1].off; - DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", - nbMatches, maxML, maxOffset, (U32)(ip-prefixStart)); + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); if (maxML > sufficient_len) { - lastSequence.litlen = litlen; - lastSequence.mlen = maxML; - lastSequence.off = maxOffset; - DEBUGLOG(6, "large match (%u>%u), immediate encoding", + lastStretch.litlen = 0; + lastStretch.mlen = maxML; + lastStretch.off = maxOffBase; + DEBUGLOG(6, "large match (%u>%u) => immediate encoding", maxML, sufficient_len); cur = 0; - last_pos = ZSTD_totalLen(lastSequence); + last_pos = maxML; goto _shortestPath; } } /* set prices for first matches starting position == 0 */ - { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 pos; + assert(opt[0].price >= 0); + { U32 pos; U32 matchNb; for (pos = 1; pos < minMatch; pos++) { - opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + opt[pos].price = ZSTD_MAX_PRICE; + opt[pos].mlen = 0; + opt[pos].litlen = litlen + pos; } for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offset = matches[matchNb].off; + U32 const offBase = matches[matchNb].off; U32 const end = matches[matchNb].len; for ( ; pos <= end ; pos++ ) { - U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); - U32 const sequencePrice = literalsPrice + matchPrice; + int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); + int const sequencePrice = opt[0].price + matchPrice; DEBUGLOG(7, "rPos:%u => set initial price : %.2f", pos, ZSTD_fCost(sequencePrice)); opt[pos].mlen = pos; - opt[pos].off = offset; - opt[pos].litlen = litlen; - opt[pos].price = sequencePrice; - } } + opt[pos].off = offBase; + opt[pos].litlen = 0; /* end of match */ + opt[pos].price = sequencePrice + LL_PRICE(0); + } + } last_pos = pos-1; + opt[pos].price = ZSTD_MAX_PRICE; } } /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { const BYTE* const inr = ip + cur; - assert(cur < ZSTD_OPT_NUM); - DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) + assert(cur <= ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur); /* Fix current position with one literal if cheaper */ - { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; + { U32 const litlen = opt[cur-1].litlen + 1; int const price = opt[cur-1].price - + ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) - + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) - - ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + + LIT_PRICE(ip+cur-1) + + LL_INCPRICE(litlen); assert(price < 1000000000); /* overflow check */ if (price <= opt[cur].price) { - DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + ZSTD_optimal_t const prevMatch = opt[cur]; + DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); - opt[cur].mlen = 0; - opt[cur].off = 0; + opt[cur] = opt[cur-1]; opt[cur].litlen = litlen; opt[cur].price = price; + if ( (optLevel >= 1) /* additional check only for higher modes */ + && (prevMatch.litlen == 0) /* replace a match */ + && (LL_INCPRICE(1) < 0) /* ll1 is cheaper than ll0 */ + && LIKELY(ip + cur < iend) + ) { + /* check next position, in case it would be cheaper */ + int with1literal = prevMatch.price + LIT_PRICE(ip+cur) + LL_INCPRICE(1); + int withMoreLiterals = price + LIT_PRICE(ip+cur) + LL_INCPRICE(litlen+1); + DEBUGLOG(7, "then at next rPos %u : match+1lit %.2f vs %ulits %.2f", + cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals)); + if ( (with1literal < withMoreLiterals) + && (with1literal < opt[cur+1].price) ) { + /* update offset history - before it disappears */ + U32 const prev = cur - prevMatch.mlen; + Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0); + assert(cur >= prevMatch.mlen); + DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !", + ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals), + newReps.rep[0], newReps.rep[1], newReps.rep[2] ); + opt[cur+1] = prevMatch; /* mlen & offbase */ + ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t)); + opt[cur+1].litlen = 1; + opt[cur+1].price = with1literal; + if (last_pos < cur+1) last_pos = cur+1; + } + } } else { - DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", - inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), - opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); + DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)", + (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); } } - /* Set the repcodes of the current position. We must do it here - * because we rely on the repcodes of the 2nd to last sequence being - * correct to set the next chunks repcodes during the backward - * traversal. + /* Offset history is not updated during match comparison. + * Do it here, now that the match is selected and confirmed. */ - ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t)); assert(cur >= opt[cur].mlen); - if (opt[cur].mlen != 0) { + if (opt[cur].litlen == 0) { + /* just finished a match => alter offset history */ U32 const prev = cur - opt[cur].mlen; - repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); - ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); - } else { - ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t)); } /* last match must start at a minimum distance of 8 from oend */ @@ -24053,37 +33825,37 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if ( (optLevel==0) /*static_test*/ && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { - DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); + DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1); continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ } - { U32 const ll0 = (opt[cur].mlen != 0); - U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; - U32 const previousPrice = opt[cur].price; - U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].litlen == 0); + int const previousPrice = opt[cur].price; + int const basePrice = previousPrice + LL_PRICE(0); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); U32 matchNb; ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(inr-istart), (U32)(iend-inr)); + (U32)(inr-istart), (U32)(iend-inr), + minMatch); if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); continue; } - { U32 const maxML = matches[nbMatches-1].len; - DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", - inr-istart, cur, nbMatches, maxML); + { U32 const longestML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u", + (int)(inr-istart), cur, nbMatches, longestML); - if ( (maxML > sufficient_len) - || (cur + maxML >= ZSTD_OPT_NUM) ) { - lastSequence.mlen = maxML; - lastSequence.off = matches[nbMatches-1].off; - lastSequence.litlen = litlen; - cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ - last_pos = cur + ZSTD_totalLen(lastSequence); - if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ + if ( (longestML > sufficient_len) + || (cur + longestML >= ZSTD_OPT_NUM) + || (ip + cur + longestML >= iend) ) { + lastStretch.mlen = longestML; + lastStretch.off = matches[nbMatches-1].off; + lastStretch.litlen = 0; + last_pos = cur + longestML; goto _shortestPath; } } @@ -24094,20 +33866,25 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; U32 mlen; - DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", - matchNb, matches[matchNb].off, lastML, litlen); + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, opt[cur].litlen); for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ U32 const pos = cur + mlen; - int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); if ((pos > last_pos) || (price < opt[pos].price)) { DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); - while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ + while (last_pos < pos) { + /* fill empty positions, for future comparisons */ + last_pos++; + opt[last_pos].price = ZSTD_MAX_PRICE; + opt[last_pos].litlen = !0; /* just needs to be != 0, to mean "not an end of match" */ + } opt[pos].mlen = mlen; opt[pos].off = offset; - opt[pos].litlen = litlen; + opt[pos].litlen = 0; opt[pos].price = price; } else { DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", @@ -24115,55 +33892,89 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ } } } } + opt[last_pos+1].price = ZSTD_MAX_PRICE; } /* for (cur = 1; cur <= last_pos; cur++) */ - lastSequence = opt[last_pos]; - cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ - assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + lastStretch = opt[last_pos]; + assert(cur >= lastStretch.mlen); + cur = last_pos - lastStretch.mlen; _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ assert(opt[0].mlen == 0); + assert(last_pos >= lastStretch.mlen); + assert(cur == last_pos - lastStretch.mlen); - /* Set the next chunk's repcodes based on the repcodes of the beginning - * of the last match, and the last sequence. This avoids us having to - * update them while traversing the sequences. - */ - if (lastSequence.mlen != 0) { - repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); - ZSTD_memcpy(rep, &reps, sizeof(reps)); + if (lastStretch.mlen==0) { + /* no solution : all matches have been converted into literals */ + assert(lastStretch.litlen == (ip - anchor) + last_pos); + ip += last_pos; + continue; + } + assert(lastStretch.off > 0); + + /* Update offset history */ + if (lastStretch.litlen == 0) { + /* finishing on a match : update offset history */ + Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t)); } else { - ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t)); + assert(cur >= lastStretch.litlen); + cur -= lastStretch.litlen; } - { U32 const storeEnd = cur + 1; + /* Let's write the shortest path solution. + * It is stored in @opt in reverse order, + * starting from @storeEnd (==cur+2), + * effectively partially @opt overwriting. + * Content is changed too: + * - So far, @opt stored stretches, aka a match followed by literals + * - Now, it will store sequences, aka literals followed by a match + */ + { U32 const storeEnd = cur + 2; U32 storeStart = storeEnd; - U32 seqPos = cur; + U32 stretchPos = cur; DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", last_pos, cur); (void)last_pos; - assert(storeEnd < ZSTD_OPT_NUM); - DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", - storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); - opt[storeEnd] = lastSequence; - while (seqPos > 0) { - U32 const backDist = ZSTD_totalLen(opt[seqPos]); + assert(storeEnd < ZSTD_OPT_SIZE); + DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off); + if (lastStretch.litlen > 0) { + /* last "sequence" is unfinished: just a bunch of literals */ + opt[storeEnd].litlen = lastStretch.litlen; + opt[storeEnd].mlen = 0; + storeStart = storeEnd-1; + opt[storeStart] = lastStretch; + } { + opt[storeEnd] = lastStretch; /* note: litlen will be fixed */ + storeStart = storeEnd; + } + while (1) { + ZSTD_optimal_t nextStretch = opt[stretchPos]; + opt[storeStart].litlen = nextStretch.litlen; + DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)", + opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off); + if (nextStretch.mlen == 0) { + /* reaching beginning of segment */ + break; + } storeStart--; - DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", - seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); - opt[storeStart] = opt[seqPos]; - seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + opt[storeStart] = nextStretch; /* note: litlen will be fixed */ + assert(nextStretch.litlen + nextStretch.mlen <= stretchPos); + stretchPos -= nextStretch.litlen + nextStretch.mlen; } /* save sequences */ - DEBUGLOG(6, "sending selected sequences into seqStore") + DEBUGLOG(6, "sending selected sequences into seqStore"); { U32 storePos; for (storePos=storeStart; storePos <= storeEnd; storePos++) { U32 const llen = opt[storePos].litlen; U32 const mlen = opt[storePos].mlen; - U32 const offCode = opt[storePos].off; + U32 const offBase = opt[storePos].off; U32 const advance = llen + mlen; - DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", - anchor - istart, (unsigned)llen, (unsigned)mlen); + DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u", + (int)(anchor - istart), (unsigned)llen, (unsigned)mlen); if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ assert(storePos == storeEnd); /* must be last sequence */ @@ -24172,11 +33983,14 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } assert(anchor + llen <= iend); - ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); anchor += advance; ip = anchor; } } + DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]); + + /* update all costs */ ZSTD_setBasePrices(optStatePtr, optLevel); } } /* while (ip < ilimit) */ @@ -24184,50 +33998,51 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ /* Return the last literals size */ return (size_t)(iend - anchor); } +#endif /* build exclusions */ +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt0( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); +} +#endif +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt2( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btopt( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_noDict); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } +#endif -/* used in 2-pass strategy */ -static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) -{ - U32 s, sum=0; - assert(ZSTD_FREQ_DIV+bonus >= 0); - for (s=0; slitSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); - optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); -} +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR /* ZSTD_initStats_ultra(): * make a first compression pass, just to seed stats with more accurate starting values. * only works on first block, with no dictionary and no ldm. - * this function cannot error, hence its contract must be respected. + * this function cannot error out, its narrow contract must be respected. */ -static void -ZSTD_initStats_ultra(ZSTD_matchState_t* ms, - seqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms, + SeqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) { U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); @@ -24238,38 +34053,36 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ - ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ - /* invalidate first scan from history */ + /* invalidate first scan from history, only keep entropy stats */ ZSTD_resetSeqStore(seqStore); ms->window.base -= srcSize; ms->window.dictLimit += (U32)srcSize; ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - /* re-inforce weight of collected statistics */ - ZSTD_upscaleStats(&ms->opt); } size_t ZSTD_compressBlock_btultra( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } size_t ZSTD_compressBlock_btultra2( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { U32 const curr = (U32)((const BYTE*)src - ms->window.base); DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); - /* 2-pass strategy: + /* 2-passes strategy: * this strategy makes a first pass over first block to collect statistics - * and seed next round's statistics with it. - * After 1st pass, function forgets everything, and starts a new block. + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. * Consequently, this can only work if no data has been previously loaded in tables, * aka, no dictionary, no prefix, no ldm preprocessing. * The compression ratio gain is generally small (~0.5% on first block), @@ -24278,42 +34091,47 @@ size_t ZSTD_compressBlock_btultra2( if ( (ms->opt.litLengthSum==0) /* first block */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ - && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ - && (srcSize > ZSTD_PREDEF_THRESHOLD) + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ ) { ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); } - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); } +#endif +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btopt_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState); -} - -size_t ZSTD_compressBlock_btultra_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_btopt_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_btultra_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) { - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); } +#endif /* note : no btultra2 variant for extDict nor dictMatchState, * because btultra2 is not meant to work with dictionaries @@ -24322,7 +34140,7 @@ size_t ZSTD_compressBlock_btultra_extDict( #ifdef ZSTD_MULTITHREAD /**** start inlining compress/zstdmt_compress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -24338,11 +34156,8 @@ size_t ZSTD_compressBlock_btultra_extDict( #endif -/* ====== Constants ====== */ -#define ZSTDMT_OVERLAPLOG_DEFAULT 0 - - /* ====== Dependencies ====== */ +/**** skipping file: ../common/allocations.h ****/ /**** skipping file: ../common/zstd_deps.h ****/ /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/pool.h ****/ @@ -24366,12 +34181,13 @@ size_t ZSTD_compressBlock_btultra_extDict( # include # include -# define DEBUG_PRINTHEX(l,p,n) { \ - unsigned debug_u; \ - for (debug_u=0; debug_u<(n); debug_u++) \ - RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ - RAWLOG(l, " \n"); \ -} +# define DEBUG_PRINTHEX(l,p,n) \ + do { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ + } while (0) static unsigned long long GetCurrentClockTimeMicroseconds(void) { @@ -24383,25 +34199,28 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) } } #define MUTEX_WAIT_TIME_DLEVEL 6 -#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ - if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ - unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ - ZSTD_pthread_mutex_lock(mutex); \ - { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ - unsigned long long const elapsedTime = (afterTime-beforeTime); \ - if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ - DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ - elapsedTime, #mutex); \ - } } \ - } else { \ - ZSTD_pthread_mutex_lock(mutex); \ - } \ -} +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) \ + do { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { \ + /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, \ + "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ + } while (0) #else # define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) -# define DEBUG_PRINTHEX(l,p,n) {} +# define DEBUG_PRINTHEX(l,p,n) do { } while (0) #endif @@ -24412,9 +34231,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) typedef struct buffer_s { void* start; size_t capacity; -} buffer_t; +} Buffer; -static const buffer_t g_nullBuffer = { NULL, 0 }; +static const Buffer g_nullBuffer = { NULL, 0 }; typedef struct ZSTDMT_bufferPool_s { ZSTD_pthread_mutex_t poolMutex; @@ -24422,19 +34241,39 @@ typedef struct ZSTDMT_bufferPool_s { unsigned totalBuffers; unsigned nbBuffers; ZSTD_customMem cMem; - buffer_t bTable[1]; /* variable size */ + Buffer* buffers; } ZSTDMT_bufferPool; -static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_customMem cMem) +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) { - unsigned const maxNbBuffers = 2*nbWorkers + 3; - ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( - sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + if (bufPool->buffers) { + unsigned u; + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->buffers[u].start); + ZSTD_customFree(bufPool->buffers[u].start, bufPool->cMem); + } + ZSTD_customFree(bufPool->buffers, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); +} + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) +{ + ZSTDMT_bufferPool* const bufPool = + (ZSTDMT_bufferPool*)ZSTD_customCalloc(sizeof(ZSTDMT_bufferPool), cMem); if (bufPool==NULL) return NULL; if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { ZSTD_customFree(bufPool, cMem); return NULL; } + bufPool->buffers = (Buffer*)ZSTD_customCalloc(maxNbBuffers * sizeof(Buffer), cMem); + if (bufPool->buffers==NULL) { + ZSTDMT_freeBufferPool(bufPool); + return NULL; + } bufPool->bufferSize = 64 KB; bufPool->totalBuffers = maxNbBuffers; bufPool->nbBuffers = 0; @@ -24442,32 +34281,19 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_custo return bufPool; } -static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) -{ - unsigned u; - DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); - if (!bufPool) return; /* compatibility with free on NULL */ - for (u=0; utotalBuffers; u++) { - DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); - ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); - } - ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); - ZSTD_customFree(bufPool, bufPool->cMem); -} - /* only works at initialization, not during compression */ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) { - size_t const poolSize = sizeof(*bufPool) - + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + size_t const poolSize = sizeof(*bufPool); + size_t const arraySize = bufPool->totalBuffers * sizeof(Buffer); unsigned u; size_t totalBufferSize = 0; ZSTD_pthread_mutex_lock(&bufPool->poolMutex); for (u=0; utotalBuffers; u++) - totalBufferSize += bufPool->bTable[u].capacity; + totalBufferSize += bufPool->buffers[u].capacity; ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - return poolSize + totalBufferSize; + return poolSize + arraySize + totalBufferSize; } /* ZSTDMT_setBufferSize() : @@ -24483,9 +34309,8 @@ static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const } -static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, U32 nbWorkers) +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) { - unsigned const maxNbBuffers = 2*nbWorkers + 3; if (srcBufPool==NULL) return NULL; if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ return srcBufPool; @@ -24494,7 +34319,7 @@ static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ ZSTDMT_bufferPool* newBufPool; ZSTDMT_freeBufferPool(srcBufPool); - newBufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); if (newBufPool==NULL) return newBufPool; ZSTDMT_setBufferSize(newBufPool, bSize); return newBufPool; @@ -24505,15 +34330,15 @@ static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, * assumption : bufPool must be valid * @return : a buffer, with start pointer and size * note: allocation may fail, in this case, start==NULL and size==0 */ -static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) { size_t const bSize = bufPool->bufferSize; DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); ZSTD_pthread_mutex_lock(&bufPool->poolMutex); if (bufPool->nbBuffers) { /* try to use an existing buffer */ - buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + Buffer const buf = bufPool->buffers[--(bufPool->nbBuffers)]; size_t const availBufferSize = buf.capacity; - bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + bufPool->buffers[bufPool->nbBuffers] = g_nullBuffer; if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { /* large enough, but not too much */ DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", @@ -24528,7 +34353,7 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); /* create new buffer */ DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); - { buffer_t buffer; + { Buffer buffer; void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); buffer.start = start; /* note : start can be NULL if malloc fails ! */ buffer.capacity = (start==NULL) ? 0 : bSize; @@ -24547,12 +34372,12 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) * @return : a buffer that is at least the buffer pool buffer size. * If a reallocation happens, the data in the input buffer is copied. */ -static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +static Buffer ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, Buffer buffer) { size_t const bSize = bufPool->bufferSize; if (buffer.capacity < bSize) { void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); - buffer_t newBuffer; + Buffer newBuffer; newBuffer.start = start; newBuffer.capacity = start == NULL ? 0 : bSize; if (start != NULL) { @@ -24568,24 +34393,34 @@ static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) #endif /* store buffer for later re-use, up to pool capacity */ -static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, Buffer buf) { DEBUGLOG(5, "ZSTDMT_releaseBuffer"); if (buf.start == NULL) return; /* compatible with release on NULL */ ZSTD_pthread_mutex_lock(&bufPool->poolMutex); if (bufPool->nbBuffers < bufPool->totalBuffers) { - bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + bufPool->buffers[bufPool->nbBuffers++] = buf; /* stored for later use */ DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); return; } ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); - /* Reached bufferPool capacity (should not happen) */ + /* Reached bufferPool capacity (note: should not happen) */ DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); ZSTD_customFree(buf.start, bufPool->cMem); } +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) + +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) /* ===== Seq Pool Wrapper ====== */ @@ -24596,23 +34431,23 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) return ZSTDMT_sizeof_bufferPool(seqPool); } -static rawSeqStore_t bufferToSeq(buffer_t buffer) +static RawSeqStore_t bufferToSeq(Buffer buffer) { - rawSeqStore_t seq = kNullRawSeqStore; + RawSeqStore_t seq = kNullRawSeqStore; seq.seq = (rawSeq*)buffer.start; seq.capacity = buffer.capacity / sizeof(rawSeq); return seq; } -static buffer_t seqToBuffer(rawSeqStore_t seq) +static Buffer seqToBuffer(RawSeqStore_t seq) { - buffer_t buffer; + Buffer buffer; buffer.start = seq.seq; buffer.capacity = seq.capacity * sizeof(rawSeq); return buffer; } -static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) { if (seqPool->bufferSize == 0) { return kNullRawSeqStore; @@ -24621,13 +34456,13 @@ static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) } #if ZSTD_RESIZE_SEQPOOL -static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +static RawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq) { return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); } #endif -static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq) { ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); } @@ -24639,7 +34474,7 @@ static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) { - ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); if (seqPool == NULL) return NULL; ZSTDMT_setNbSeq(seqPool, 0); return seqPool; @@ -24652,7 +34487,7 @@ static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) { - return ZSTDMT_expandBufferPool(pool, nbWorkers); + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); } @@ -24664,16 +34499,20 @@ typedef struct { int totalCCtx; int availCCtx; ZSTD_customMem cMem; - ZSTD_CCtx* cctx[1]; /* variable size */ + ZSTD_CCtx** cctxs; } ZSTDMT_CCtxPool; -/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +/* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) { - int cid; - for (cid=0; cidtotalCCtx; cid++) - ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + if (!pool) return; ZSTD_pthread_mutex_destroy(&pool->poolMutex); + if (pool->cctxs) { + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctxs[cid]); /* free compatible with NULL */ + ZSTD_customFree(pool->cctxs, pool->cMem); + } ZSTD_customFree(pool, pool->cMem); } @@ -24682,19 +34521,24 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, ZSTD_customMem cMem) { - ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( - sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); + ZSTDMT_CCtxPool* const cctxPool = + (ZSTDMT_CCtxPool*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtxPool), cMem); assert(nbWorkers > 0); if (!cctxPool) return NULL; if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { ZSTD_customFree(cctxPool, cMem); return NULL; } - cctxPool->cMem = cMem; cctxPool->totalCCtx = nbWorkers; + cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem); + if (!cctxPool->cctxs) { + ZSTDMT_freeCCtxPool(cctxPool); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ - cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); - if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); return cctxPool; } @@ -24716,16 +34560,16 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) { ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); { unsigned const nbWorkers = cctxPool->totalCCtx; - size_t const poolSize = sizeof(*cctxPool) - + (nbWorkers-1) * sizeof(ZSTD_CCtx*); - unsigned u; + size_t const poolSize = sizeof(*cctxPool); + size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*); size_t totalCCtxSize = 0; + unsigned u; for (u=0; ucctx[u]); + totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctxs[u]); } ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); assert(nbWorkers > 0); - return poolSize + totalCCtxSize; + return poolSize + arraySize + totalCCtxSize; } } @@ -24735,7 +34579,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); if (cctxPool->availCCtx) { cctxPool->availCCtx--; - { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + { ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx]; ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); return cctx; } } @@ -24749,7 +34593,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) if (cctx==NULL) return; /* compatibility with release on NULL */ ZSTD_pthread_mutex_lock(&pool->poolMutex); if (pool->availCCtx < pool->totalCCtx) - pool->cctx[pool->availCCtx++] = cctx; + pool->cctxs[pool->availCCtx++] = cctx; else { /* pool overflow : should not happen, since totalCCtx==nbWorkers */ DEBUGLOG(4, "CCtx pool overflow : free cctx"); @@ -24763,7 +34607,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) typedef struct { void const* start; size_t size; -} range_t; +} Range; typedef struct { /* All variables in the struct are protected by mutex. */ @@ -24779,10 +34623,10 @@ typedef struct { ZSTD_pthread_mutex_t ldmWindowMutex; ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ -} serialState_t; +} SerialState; static int -ZSTDMT_serialState_reset(serialState_t* serialState, +ZSTDMT_serialState_reset(SerialState* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, size_t jobSize, @@ -24790,7 +34634,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, ZSTD_dictContentType_e dictContentType) { /* Adjust parameters */ - if (params.ldmParams.enableLdm) { + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); @@ -24801,7 +34645,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, serialState->nextJobID = 0; if (params.fParams.checksumFlag) XXH64_reset(&serialState->xxhState, 0); - if (params.ldmParams.enableLdm) { + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { ZSTD_customMem cMem = params.customMem; unsigned const hashLog = params.ldmParams.hashLog; size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); @@ -24835,7 +34679,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, if (dictSize > 0) { if (dictContentType == ZSTD_dct_rawContent) { BYTE const* const dictEnd = (const BYTE*)dict + dictSize; - ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); } else { @@ -24852,7 +34696,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, return 0; } -static int ZSTDMT_serialState_init(serialState_t* serialState) +static int ZSTDMT_serialState_init(SerialState* serialState) { int initError = 0; ZSTD_memset(serialState, 0, sizeof(*serialState)); @@ -24863,7 +34707,7 @@ static int ZSTDMT_serialState_init(serialState_t* serialState) return initError; } -static void ZSTDMT_serialState_free(serialState_t* serialState) +static void ZSTDMT_serialState_free(SerialState* serialState) { ZSTD_customMem cMem = serialState->params.customMem; ZSTD_pthread_mutex_destroy(&serialState->mutex); @@ -24874,9 +34718,10 @@ static void ZSTDMT_serialState_free(serialState_t* serialState) ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); } -static void ZSTDMT_serialState_update(serialState_t* serialState, - ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, - range_t src, unsigned jobID) +static void +ZSTDMT_serialState_genSequences(SerialState* serialState, + RawSeqStore_t* seqStore, + Range src, unsigned jobID) { /* Wait for our turn */ ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); @@ -24887,14 +34732,15 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, /* A future job may error and skip our job */ if (serialState->nextJobID == jobID) { /* It is now our turn, do any processing necessary */ - if (serialState->params.ldmParams.enableLdm) { + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { size_t error; - assert(seqStore.seq != NULL && seqStore.pos == 0 && - seqStore.size == 0 && seqStore.capacity > 0); + DEBUGLOG(6, "ZSTDMT_serialState_genSequences: LDM update"); + assert(seqStore->seq != NULL && seqStore->pos == 0 && + seqStore->size == 0 && seqStore->capacity > 0); assert(src.size <= serialState->params.jobSize); - ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); error = ZSTD_ldm_generateSequences( - &serialState->ldmState, &seqStore, + &serialState->ldmState, seqStore, &serialState->params.ldmParams, src.start, src.size); /* We provide a large enough buffer to never fail. */ assert(!ZSTD_isError(error)); (void)error; @@ -24913,17 +34759,22 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, serialState->nextJobID++; ZSTD_pthread_cond_broadcast(&serialState->cond); ZSTD_pthread_mutex_unlock(&serialState->mutex); +} - if (seqStore.size > 0) { - size_t const err = ZSTD_referenceExternalSequences( - jobCCtx, seqStore.seq, seqStore.size); - assert(serialState->params.ldmParams.enableLdm); - assert(!ZSTD_isError(err)); - (void)err; +static void +ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an assert() check */ + ZSTD_CCtx* jobCCtx, + const RawSeqStore_t* seqStore) +{ + if (seqStore->size > 0) { + DEBUGLOG(5, "ZSTDMT_serialState_applySequences: uploading %u external sequences", (unsigned)seqStore->size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); (void)serialState; + assert(jobCCtx); + ZSTD_referenceExternalSequences(jobCCtx, seqStore->seq, seqStore->size); } } -static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, +static void ZSTDMT_serialState_ensureFinished(SerialState* serialState, unsigned jobID, size_t cSize) { ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); @@ -24947,36 +34798,37 @@ static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, /* ===== Worker thread ===== */ /* ------------------------------------------ */ -static const range_t kNullRange = { NULL, 0 }; +static const Range kNullRange = { NULL, 0 }; typedef struct { - size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ - size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ - ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ - ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ - ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ - ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ - ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ - serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ - buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ - range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ - range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ - unsigned jobID; /* set by mtctx, then read by worker => no barrier */ - unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ - unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ - ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ - const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ - unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ - size_t dstFlushed; /* used only by mtctx */ - unsigned frameChecksumNeeded; /* used only by mtctx */ + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + SerialState* serial; /* Thread-safe - used by mtctx and (all) workers */ + Buffer dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + Range prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + Range src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ } ZSTDMT_jobDescription; -#define JOB_ERROR(e) { \ - ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ - job->cSize = e; \ - ZSTD_pthread_mutex_unlock(&job->job_mutex); \ - goto _endJob; \ -} +#define JOB_ERROR(e) \ + do { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ + } while (0) /* ZSTDMT_compressionJob() is a POOL_function type */ static void ZSTDMT_compressionJob(void* jobDescription) @@ -24984,10 +34836,11 @@ static void ZSTDMT_compressionJob(void* jobDescription) ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); - rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); - buffer_t dstBuff = job->dstBuff; + RawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + Buffer dstBuff = job->dstBuff; size_t lastCBlockSize = 0; + DEBUGLOG(5, "ZSTDMT_compressionJob: job %u", job->jobID); /* resources */ if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ @@ -24995,7 +34848,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ } - if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL) + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) JOB_ERROR(ERROR(memory_allocation)); /* Don't compute the checksum for chunks, since we compute it externally, @@ -25003,40 +34856,49 @@ static void ZSTDMT_compressionJob(void* jobDescription) */ if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; /* Don't run LDM for the chunks, since we handle it externally */ - jobParams.ldmParams.enableLdm = 0; + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; /* Correct nbWorkers to 0. */ jobParams.nbWorkers = 0; /* init */ + + /* Perform serial step as early as possible */ + ZSTDMT_serialState_genSequences(job->serial, &rawSeqStore, job->src, job->jobID); + if (job->cdict) { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); assert(job->firstJob); /* only allowed for first job */ if (ZSTD_isError(initError)) JOB_ERROR(initError); - } else { /* srcStart points at reloaded section */ + } else { U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } + DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size); { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, - job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, ZSTD_dtlm_fast, NULL, /*cdict*/ &jobParams, pledgedSrcSize); if (ZSTD_isError(initError)) JOB_ERROR(initError); } } - /* Perform serial step as early as possible, but after CCtx initialization */ - ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + /* External Sequences can only be applied after CCtx initialization */ + ZSTDMT_serialState_applySequences(job->serial, cctx, &rawSeqStore); if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ - size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); if (ZSTD_isError(hSize)) JOB_ERROR(hSize); DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); ZSTD_invalidateRepCodes(cctx); } - /* compress */ + /* compress the entire job by smaller chunks, for better granularity */ { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); const BYTE* ip = (const BYTE*) job->src.start; @@ -25048,7 +34910,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); assert(job->cSize == 0); for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { - size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize); + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); if (ZSTD_isError(cSize)) JOB_ERROR(cSize); ip += chunkSize; op += cSize; assert(op < oend); @@ -25068,11 +34930,17 @@ static void ZSTDMT_compressionJob(void* jobDescription) size_t const lastBlockSize1 = job->src.size & (chunkSize-1); size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; size_t const cSize = (job->lastJob) ? - ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) : - ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize); + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); if (ZSTD_isError(cSize)) JOB_ERROR(cSize); lastCBlockSize = cSize; } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } ZSTD_CCtx_trace(cctx, 0); _endJob: @@ -25098,10 +34966,10 @@ _endJob: /* ------------------------------------------ */ typedef struct { - range_t prefix; /* read-only non-owned prefix buffer */ - buffer_t buffer; + Range prefix; /* read-only non-owned prefix buffer */ + Buffer buffer; size_t filled; -} inBuff_t; +} InBuff_t; typedef struct { BYTE* buffer; /* The round input buffer. All jobs get references @@ -25115,17 +34983,26 @@ typedef struct { * the inBuff is sent to the worker thread. * pos <= capacity. */ -} roundBuff_t; +} RoundBuff_t; -static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; +static const RoundBuff_t kNullRoundBuff = {NULL, 0, 0}; #define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ - inBuff_t inBuff; - roundBuff_t roundBuff; - serialState_t serial; - rsyncState_t rsync; + InBuff_t inBuff; + RoundBuff_t roundBuff; + SerialState serial; + RSyncState_t rsync; unsigned jobIDMask; unsigned doneJobID; unsigned nextJobID; @@ -25240,7 +35117,7 @@ MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ mtctx->jobIDMask = nbJobs - 1; - mtctx->bufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); initError = ZSTDMT_serialState_init(&mtctx->serial); @@ -25343,7 +35220,7 @@ static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) { if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); - mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); if (mtctx->bufPool == NULL) return ERROR(memory_allocation); mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); @@ -25386,7 +35263,7 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) { unsigned jobNb; unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", - mtctx->doneJobID, lastJobNb, mtctx->jobReady) + mtctx->doneJobID, lastJobNb, mtctx->jobReady); for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { unsigned const wJobID = jobNb & mtctx->jobIDMask; ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; @@ -25448,7 +35325,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) { unsigned jobLog; - if (params->ldmParams.enableLdm) { + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on cycleLog instead. */ @@ -25492,7 +35369,7 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); assert(0 <= overlapRLog && overlapRLog <= 8); - if (params->ldmParams.enableLdm) { + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. @@ -25525,13 +35402,11 @@ size_t ZSTDMT_initCStream_internal( /* init */ if (params.nbWorkers != mtctx->params.nbWorkers) - FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, (unsigned)params.nbWorkers) , ""); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; - DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); - if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ ZSTDMT_waitForAllJobsCompleted(mtctx); ZSTDMT_releaseAllJobResources(mtctx); @@ -25540,15 +35415,14 @@ size_t ZSTDMT_initCStream_internal( mtctx->params = params; mtctx->frameContentSize = pledgedSrcSize; + ZSTD_freeCDict(mtctx->cdictLocal); if (dict) { - ZSTD_freeCDict(mtctx->cdictLocal); mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ params.cParams, mtctx->cMem); mtctx->cdict = mtctx->cdictLocal; if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); } else { - ZSTD_freeCDict(mtctx->cdictLocal); mtctx->cdictLocal = NULL; mtctx->cdict = cdict; } @@ -25563,9 +35437,11 @@ size_t ZSTDMT_initCStream_internal( if (params.rsyncable) { /* Aim for the targetsectionSize as the average job size. */ - U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); - U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20; - assert(jobSizeMB >= 1); + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); DEBUGLOG(4, "rsyncLog = %u", rsyncBits); mtctx->rsync.hash = 0; mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; @@ -25577,7 +35453,7 @@ size_t ZSTDMT_initCStream_internal( ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); { /* If ldm is enabled we need windowSize space. */ - size_t const windowSize = mtctx->params.ldmParams.enableLdm ? (1U << mtctx->params.cParams.windowLog) : 0; + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; /* Two buffers of slack, plus extra space for the overlap * This is the minimum slack that LDM works with. One extra because * flush might waste up to targetSectionSize-1 bytes. Another extra @@ -25612,9 +35488,32 @@ size_t ZSTDMT_initCStream_internal( mtctx->allJobsCompleted = 0; mtctx->consumed = 0; mtctx->produced = 0; + + /* update dictionary */ + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = NULL; + if (dict) { + if (dictContentType == ZSTD_dct_rawContent) { + mtctx->inBuff.prefix.start = (const BYTE*)dict; + mtctx->inBuff.prefix.size = dictSize; + } else { + /* note : a loadPrefix becomes an internal CDict */ + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, dictContentType, + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } + } else { + mtctx->cdict = cdict; + } + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, dict, dictSize, dictContentType)) return ERROR(memory_allocation); + + return 0; } @@ -25681,7 +35580,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS mtctx->roundBuff.pos += srcSize; mtctx->inBuff.buffer = g_nullBuffer; mtctx->inBuff.filled = 0; - /* Set the prefix */ + /* Set the prefix for next job */ if (!endFrame) { size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; @@ -25818,12 +35717,17 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u * If the data of the first job is broken up into two segments, we cover both * sections. */ -static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) { unsigned const firstJobID = mtctx->doneJobID; unsigned const lastJobID = mtctx->nextJobID; unsigned jobID; + /* no need to check during first round */ + size_t roundBuffCapacity = mtctx->roundBuff.capacity; + size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize; + if (lastJobID < nbJobs1stRoundMin) return kNullRange; + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { unsigned const wJobID = jobID & mtctx->jobIDMask; size_t consumed; @@ -25833,7 +35737,7 @@ static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); if (consumed < mtctx->jobs[wJobID].src.size) { - range_t range = mtctx->jobs[wJobID].prefix; + Range range = mtctx->jobs[wJobID].prefix; if (range.size == 0) { /* Empty prefix */ range = mtctx->jobs[wJobID].src; @@ -25849,26 +35753,30 @@ static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) /** * Returns non-zero iff buffer and range overlap. */ -static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +static int ZSTDMT_isOverlapped(Buffer buffer, Range range) { BYTE const* const bufferStart = (BYTE const*)buffer.start; - BYTE const* const bufferEnd = bufferStart + buffer.capacity; BYTE const* const rangeStart = (BYTE const*)range.start; - BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart; if (rangeStart == NULL || bufferStart == NULL) return 0; - /* Empty ranges cannot overlap */ - if (bufferStart == bufferEnd || rangeStart == rangeEnd) - return 0; - return bufferStart < rangeEnd && rangeStart < bufferEnd; + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; + + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } } -static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window) { - range_t extDict; - range_t prefix; + Range extDict; + Range prefix; DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); extDict.start = window.dictBase + window.lowLimit; @@ -25887,9 +35795,9 @@ static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) || ZSTDMT_isOverlapped(buffer, prefix); } -static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer) { - if (mtctx->params.ldmParams.enableLdm) { + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); DEBUGLOG(5, "source [0x%zx, 0x%zx)", @@ -25912,16 +35820,16 @@ static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) */ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) { - range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + Range const inUse = ZSTDMT_getInputDataInUse(mtctx); size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; - size_t const target = mtctx->targetSectionSize; - buffer_t buffer; + size_t const spaceNeeded = mtctx->targetSectionSize; + Buffer buffer; DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); assert(mtctx->inBuff.buffer.start == NULL); - assert(mtctx->roundBuff.capacity >= target); + assert(mtctx->roundBuff.capacity >= spaceNeeded); - if (spaceLeft < target) { + if (spaceLeft < spaceNeeded) { /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. * Simply copy the prefix to the beginning in that case. */ @@ -25940,7 +35848,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) mtctx->roundBuff.pos = prefixSize; } buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; - buffer.capacity = target; + buffer.capacity = spaceNeeded; if (ZSTDMT_isOverlapped(buffer, inUse)) { DEBUGLOG(5, "Waiting for buffer..."); @@ -25967,7 +35875,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) typedef struct { size_t toLoad; /* The number of bytes to load from the input. */ int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ -} syncPoint_t; +} SyncPoint; /** * Searches through the input for a synchronization point. If one is found, we @@ -25975,14 +35883,14 @@ typedef struct { * Otherwise, we will load as many bytes as possible and instruct the caller * to continue as normal. */ -static syncPoint_t +static SyncPoint findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) { BYTE const* const istart = (BYTE const*)input.src + input.pos; U64 const primePower = mtctx->rsync.primePower; U64 const hitMask = mtctx->rsync.hitMask; - syncPoint_t syncPoint; + SyncPoint syncPoint; U64 hash; BYTE const* prev; size_t pos; @@ -25992,6 +35900,11 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) if (!mtctx->params.rsyncable) /* Rsync is disabled. */ return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) /* Not enough to compute the hash. * We will miss any synchronization points in this RSYNC_LENGTH byte @@ -26002,10 +35915,28 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) */ return syncPoint; /* Initialize the loop variables. */ - if (mtctx->inBuff.filled >= RSYNC_LENGTH) { - /* We have enough bytes buffered to initialize the hash. + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and have processed enough bytes to find a sync point. * Start scanning at the beginning of the input. */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); pos = 0; prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); @@ -26019,16 +35950,6 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) syncPoint.flush = 1; return syncPoint; } - } else { - /* We don't have enough bytes buffered to initialize the hash, but - * we know we have at least RSYNC_LENGTH bytes total. - * Start scanning after the first RSYNC_LENGTH bytes less the bytes - * already buffered. - */ - pos = RSYNC_LENGTH - mtctx->inBuff.filled; - prev = (BYTE const*)mtctx->inBuff.buffer.start - pos; - hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled); - hash = ZSTD_rollingHash_append(hash, istart, pos); } /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll * through the input. If we hit a synchronization point, then cut the @@ -26038,16 +35959,24 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) * then a block will be emitted anyways, but this is okay, since if we * are already synchronized we will remain synchronized. */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); for (; pos < syncPoint.toLoad; ++pos) { BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; - /* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */ + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); if ((hash & hitMask) == hitMask) { syncPoint.toLoad = pos + 1; syncPoint.flush = 1; + ++pos; /* for assert */ break; } } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); return syncPoint; } @@ -26093,7 +36022,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); } if (mtctx->inBuff.buffer.start != NULL) { - syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + SyncPoint const syncPoint = findSynchronizationPoint(mtctx, *input); if (syncPoint.flush && endOp == ZSTD_e_continue) { endOp = ZSTD_e_flush; } @@ -26140,7 +36069,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, /* ****************************************************************** * huff0 huffman decoder, * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -26158,14 +36087,27 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, /**** skipping file: ../common/compiler.h ****/ /**** skipping file: ../common/bitstream.h ****/ /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/bits.h ****/ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 /* ************************************************************** * Macros ****************************************************************/ +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 +#endif + /* These two optional macros force the use one way or another of the two * Huffman decompression implementations. You can't force in both directions * at the same time. @@ -26175,6 +36117,28 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, #error "Cannot force the use of the X1 and X2 decoders at the same time!" #endif +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif /* ************************************************************** * Error Management @@ -26192,6 +36156,11 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, /* ************************************************************** * BMI2 Variant Wrappers ****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + #if DYNAMIC_BMI2 #define HUF_DGEN(fn) \ @@ -26204,7 +36173,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ \ - static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ void* dst, size_t dstSize, \ const void* cSrc, size_t cSrcSize, \ const HUF_DTable* DTable) \ @@ -26213,9 +36182,9 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, } \ \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - if (bmi2) { \ + if (flags & HUF_flags_bmi2) { \ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ @@ -26225,9 +36194,9 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, #define HUF_DGEN(fn) \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - (void)bmi2; \ + (void)flags; \ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ } @@ -26246,13 +36215,186 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) return dtd; } +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const istart = (const BYTE*)src; + + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) + #ifndef HUF_FORCE_DECOMPRESS_X2 /*-***************************/ /* single-symbol decoding */ /*-***************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ /** * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at @@ -26261,14 +36403,45 @@ typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decodi static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { U64 D4; if (MEM_isLittleEndian()) { - D4 = symbol + (nbBits << 8); + D4 = (U64)((symbol << 8) + nbBits); } else { - D4 = (symbol << 8) + nbBits; + D4 = (U64)(symbol + (nbBits << 8)); } + assert(D4 < (1U << 16)); D4 *= 0x0001000100010001ULL; return D4; } +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + typedef struct { U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; @@ -26277,13 +36450,7 @@ typedef struct { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; } HUF_ReadDTableX1_Workspace; - -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) { U32 tableLog = 0; U32 nbSymbols = 0; @@ -26298,11 +36465,15 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); if (HUF_isError(iSize)) return iSize; + /* Table header */ { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ dtd.tableType = 0; dtd.tableLog = (BYTE)tableLog; @@ -26321,9 +36492,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr * rankStart[0] is not filled because there are no entries in the table for * weight 0. */ - { - int n; - int nextRankStart = 0; + { int n; + U32 nextRankStart = 0; int const unroll = 4; int const nLimit = (int)nbSymbols - unroll + 1; for (n=0; n<(int)tableLog+1; n++) { @@ -26346,14 +36516,13 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr /* fill DTable * We fill all entries of each weight in order. - * That way length is a constant for each iteration of the outter loop. + * That way length is a constant for each iteration of the outer loop. * We can switch based on the length to a different inner loop which is * optimized for that particular case. */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; for (w=1; wrankVal[w]; int const length = (1 << w) >> 1; @@ -26427,15 +36596,19 @@ HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog } #define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) -#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) -#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) HINT_INLINE size_t HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) @@ -26443,11 +36616,15 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons BYTE* const pStart = p; /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_1(p, bitDPtr); - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); } /* [0-3] symbols remaining */ @@ -26459,7 +36636,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons while (p < pEnd) HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - return pEnd-pStart; + return (size_t)(pEnd-pStart); } FORCE_INLINE_TEMPLATE size_t @@ -26469,7 +36646,7 @@ HUF_decompress1X1_usingDTable_internal_body( const HUF_DTable* DTable) { BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); const void* dtPtr = DTable + 1; const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; BIT_DStream_t bitD; @@ -26485,6 +36662,10 @@ HUF_decompress1X1_usingDTable_internal_body( return dstSize; } +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X1_usingDTable_internal_body( void* dst, size_t dstSize, @@ -26493,6 +36674,7 @@ HUF_decompress4X1_usingDTable_internal_body( { /* Check */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; @@ -26527,33 +36709,37 @@ HUF_decompress4X1_usingDTable_internal_body( U32 endSignal = 1; if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ - for ( ; (endSignal) & (op4 < olimit) ; ) { - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_1(op1, &bitD1); - HUF_DECODE_SYMBOLX1_1(op2, &bitD2); - HUF_DECODE_SYMBOLX1_1(op3, &bitD3); - HUF_DECODE_SYMBOLX1_1(op4, &bitD4); - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_0(op1, &bitD1); - HUF_DECODE_SYMBOLX1_0(op2, &bitD2); - HUF_DECODE_SYMBOLX1_0(op3, &bitD3); - HUF_DECODE_SYMBOLX1_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } } /* check corruption */ @@ -26579,74 +36765,250 @@ HUF_decompress4X1_usingDTable_internal_body( } } +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} HUF_DGEN(HUF_decompress1X1_usingDTable_internal) -HUF_DGEN(HUF_decompress4X1_usingDTable_internal) - - -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) { - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } - -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} - -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); -} - - #endif /* HUF_FORCE_DECOMPRESS_X2 */ @@ -26657,188 +37019,308 @@ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* *************************/ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; +typedef struct { BYTE symbol; } sortedSymbol_t; typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} /* HUF_fillDTableX2Level2() : * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, U32 nbBitsBaseline, U16 baseSeq) { - HUF_DEltX2 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } } - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } } - static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) { - U32 rankVal[HUF_TABLELOG_MAX + 1]; + U32* const rankVal = rankValOrigin[0]; const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; + int w; + int const wEnd = (int)maxWeight + 1; - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; - /* fill DTable */ - for (s=0; s= minBits) { /* enough room for a second symbol */ - U32 sortedRank; + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); int minWeight = nbBits + scaleLog; + int s; if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } } else { - HUF_DEltX2 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } } } +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { - U32 tableLog, maxW, sizeOfSort, nbSymbols; + U32 tableLog, maxW, nbSymbols; DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; + U32 maxTableLog = dtd.maxTableLog; size_t iSize; void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; U32 *rankStart; - rankValCol_t* rankVal; - U32* rankStats; - U32* rankStart0; - sortedSymbol_t* sortedSymbol; - BYTE* weightList; - size_t spaceUsed32 = 0; + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; - rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); - spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); - if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); - - rankStart = rankStart0 + 1; - ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); if (HUF_isError(iSize)) return iSize; /* check result */ if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ /* Get start index of each weight */ { U32 w, nextRankStart = 0; for (w=1; wrankStats[w]; rankStart[w] = curr; } rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; + rankStart[maxW+1] = nextRankStart; } /* sort symbols by weight */ { U32 s; for (s=0; sweightList[s]; U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; + wksp->sortedSymbol[r].symbol = (BYTE)s; } rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; + { U32* const rankVal0 = wksp->rankVal[0]; { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 nextRankVal = 0; U32 w; for (w=1; wrankStats[w] << (w+rescale); rankVal0[w] = curr; } } { U32 const minBits = tableLog+1 - maxW; U32 consumed; for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { - U32* const rankValPtr = rankVal[consumed]; + U32* const rankValPtr = wksp->rankVal[consumed]; U32 w; for (w = 1; w < maxW+1; w++) { rankValPtr[w] = rankVal0[w] >> consumed; } } } } HUF_fillDTableX2(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, tableLog+1); dtd.tableLog = (BYTE)maxTableLog; @@ -26852,7 +37334,7 @@ FORCE_INLINE_TEMPLATE U32 HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) { size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 2); + ZSTD_memcpy(op, &dt[val].sequence, 2); BIT_skipBits(DStream, dt[val].nbBits); return dt[val].length; } @@ -26861,28 +37343,34 @@ FORCE_INLINE_TEMPLATE U32 HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) { size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { BIT_skipBits(DStream, dt[val].nbBits); if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); - } } + } + } return 1; } #define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, @@ -26891,19 +37379,37 @@ HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, BYTE* const pStart = p; /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); } /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } if (p < pEnd) p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); @@ -26924,7 +37430,7 @@ HUF_decompress1X2_usingDTable_internal_body( /* decode */ { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; DTableDesc const dtd = HUF_getDTableDesc(DTable); @@ -26938,6 +37444,10 @@ HUF_decompress1X2_usingDTable_internal_body( return dstSize; } +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X2_usingDTable_internal_body( void* dst, size_t dstSize, @@ -26945,6 +37455,7 @@ HUF_decompress4X2_usingDTable_internal_body( const HUF_DTable* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; @@ -26978,58 +37489,62 @@ HUF_decompress4X2_usingDTable_internal_body( DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); /* 16-32 symbols per loop (4-8 symbols per stream) */ - for ( ; (endSignal) & (op4 < olimit); ) { + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { #if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; #else - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = (U32)LIKELY( - (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); #endif + } } /* check corruption */ @@ -27053,68 +37568,287 @@ HUF_decompress4X2_usingDTable_internal_body( } } -HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -HUF_DGEN(HUF_decompress4X2_usingDTable_internal) +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif -size_t HUF_decompress1X2_usingDTable( +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } + +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; } +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); } - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - - #endif /* HUF_FORCE_DECOMPRESS_X1 */ @@ -27122,66 +37856,28 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* Universal decompression selectors */ /* ***********************************/ -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = { /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ }; #endif @@ -27208,42 +37904,15 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) U32 const D256 = (U32)(dstSize >> 8); U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ return DTime1 < DTime0; } #endif } - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, - size_t dstSize, const void* cSrc, - size_t cSrcSize, void* workSpace, - size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#endif - } -} - size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -27256,71 +37925,71 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, (void)algoNb; assert(algoNb == 0); return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #else return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): + cSrcSize, workSpace, wkspSize, flags): HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #endif } } -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -27330,167 +37999,21 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds #if defined(HUF_FORCE_DECOMPRESS_X1) (void)algoNb; assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #endif } } - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif - } -} - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} - -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -#endif /**** ended inlining decompress/huf_decompress.c ****/ /**** start inlining decompress/zstd_ddict.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -27505,16 +38028,16 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, /*-******************************************************* * Dependencies *********************************************************/ +/**** skipping file: ../common/allocations.h ****/ /**** skipping file: ../common/zstd_deps.h ****/ /**** skipping file: ../common/cpu.h ****/ /**** skipping file: ../common/mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** start inlining zstd_decompress_internal.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -27536,7 +38059,6 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, *********************************************************/ /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ @@ -27556,7 +38078,7 @@ static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; -static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -27591,12 +38113,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = { #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef struct { ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ U32 rep[ZSTD_REP_NUM]; U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; } ZSTD_entropyDTables_t; @@ -27622,6 +38145,22 @@ typedef struct { size_t ddictPtrCount; } ZSTD_DDictHashSet; +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -27635,7 +38174,7 @@ struct ZSTD_DCtx_s const void* virtualStart; /* virtual start of previous segment if it was just before current one */ const void* dictEnd; /* end of previous segment */ size_t expected; - ZSTD_frameHeader fParams; + ZSTD_FrameHeader fParams; U64 processedCSize; U64 decodedSize; blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ @@ -27652,7 +38191,10 @@ struct ZSTD_DCtx_s size_t litSize; size_t rleSize; size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif /* dictionary */ ZSTD_DDict* ddictLocal; @@ -27662,6 +38204,8 @@ struct ZSTD_DCtx_s ZSTD_dictUses_e dictUses; ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; /* streaming */ ZSTD_dStreamStage streamStage; @@ -27674,16 +38218,21 @@ struct ZSTD_DCtx_s size_t outStart; size_t outEnd; size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; +#endif U32 hostageByte; int noForwardProgress; ZSTD_bufferMode_e outBufferMode; ZSTD_outBuffer expectedOutBuffer; /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t oversizedDuration; @@ -27699,6 +38248,14 @@ struct ZSTD_DCtx_s #endif }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} /*-******************************************************* * Shared internal functions @@ -27722,7 +38279,7 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); /**** ended inlining zstd_decompress_internal.h ****/ /**** start inlining zstd_ddict.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -27768,1373 +38325,7 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /**** ended inlining zstd_ddict.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** start inlining ../legacy/zstd_legacy.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_LEGACY_H -#define ZSTD_LEGACY_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/error_private.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ - -#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) -# undef ZSTD_LEGACY_SUPPORT -# define ZSTD_LEGACY_SUPPORT 8 -#endif - -#if (ZSTD_LEGACY_SUPPORT <= 1) -/**** start inlining zstd_v01.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V01_H_28739879432 -#define ZSTD_V01_H_28739879432 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error -*/ -unsigned ZSTDv01_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; -ZSTDv01_Dctx* ZSTDv01_createDCtx(void); -size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); - -size_t ZSTDv01_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); - -size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); -size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ -#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V01_H_28739879432 */ -/**** ended inlining zstd_v01.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) -/**** start inlining zstd_v02.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V02_H_4174539423 -#define ZSTD_V02_H_4174539423 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error -*/ -unsigned ZSTDv02_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx; -ZSTDv02_Dctx* ZSTDv02_createDCtx(void); -size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx); - -size_t ZSTDv02_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx); - -size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx); -size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V02_H_4174539423 */ -/**** ended inlining zstd_v02.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) -/**** start inlining zstd_v03.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V03_H_298734209782 -#define ZSTD_V03_H_298734209782 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - - /** -ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error -*/ -unsigned ZSTDv03_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx; -ZSTDv03_Dctx* ZSTDv03_createDCtx(void); -size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx); - -size_t ZSTDv03_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx); - -size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx); -size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V03_H_298734209782 */ -/**** ended inlining zstd_v03.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) -/**** start inlining zstd_v04.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V04_H_91868324769238 -#define ZSTD_V04_H_91868324769238 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error -*/ -unsigned ZSTDv04_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx; -ZSTDv04_Dctx* ZSTDv04_createDCtx(void); -size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx); - -size_t ZSTDv04_decompressDCtx(ZSTDv04_Dctx* dctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - -/* ************************************* -* Direct Streaming -***************************************/ -size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx); - -size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx); -size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - - -/* ************************************* -* Buffered Streaming -***************************************/ -typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx; -ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void); -size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx); - -size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx); -size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize); - -size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); - -/** ************************************************ -* Streaming decompression -* -* A ZBUFF_DCtx object is required to track streaming operation. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. -* -* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() -* It must be the same content as the one set during compression phase. -* Dictionary content must remain accessible during the decompression process. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ -unsigned ZBUFFv04_isError(size_t errorCode); -const char* ZBUFFv04_getErrorName(size_t errorCode); - - -/** The below functions provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFFv04_recommendedDInSize(void); -size_t ZBUFFv04_recommendedDOutSize(void); - - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V04_H_91868324769238 */ -/**** ended inlining zstd_v04.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) -/**** start inlining zstd_v05.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv05_H -#define ZSTDv05_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-************************************* -* Dependencies -***************************************/ -#include /* size_t */ -/**** skipping file: ../common/mem.h ****/ - - -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv05_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ -size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - - /** - ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/* ************************************* -* Helper functions -***************************************/ -/* Error Management */ -unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */ - - -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; -ZSTDv05_DCtx* ZSTDv05_createDCtx(void); -size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv05_decompressDCtx() : -* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ -size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Simple Dictionary API -*************************/ -/*! ZSTDv05_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ -size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - -/*-************************ -* Advanced Streaming API -***************************/ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; -typedef struct { - U64 srcSize; - U32 windowLog; /* the only useful information to retrieve */ - U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; -} ZSTDv05_parameters; -size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); - -size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); -void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); -size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); -size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* ZBUFF API -*************************/ -typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; -ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); -size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); - -size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); -size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); - -size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression -* -* A ZBUFFv05_DCtx object is required to track streaming operations. -* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources. -* Use ZBUFFv05_decompressInit() to start a new decompression operation, -* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv05_DCtx objects can be reused multiple times. -* -* Use ZBUFFv05_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFFv05_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize() -* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -unsigned ZBUFFv05_isError(size_t errorCode); -const char* ZBUFFv05_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, and tend to offer better latency */ -size_t ZBUFFv05_recommendedDInSize(void); -size_t ZBUFFv05_recommendedDOutSize(void); - - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */ - - - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv0505_H */ -/**** ended inlining zstd_v05.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) -/**** start inlining zstd_v06.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv06_H -#define ZSTDv06_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv06_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) -# define ZSTDLIBv06_API __declspec(dllexport) -#else -# define ZSTDLIBv06_API -#endif - - -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv06_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/** -ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/* ************************************* -* Helper functions -***************************************/ -ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ - -/* Error Management */ -ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ - - -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; -ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); -ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv06_decompressDCtx() : -* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTDv06_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ -ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************ -* Advanced Streaming API -***************************/ -struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; -typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; - -ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); - -ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - - -/* ************************************* -* ZBUFF API -***************************************/ - -typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; -ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); -ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv06_DCtx object is required to track streaming operations. -* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources. -* Use ZBUFFv06_decompressInit() to start a new decompression operation, -* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv06_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv06_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv06_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize() -* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv06_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode); -ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void); -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */ - - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv06_BUFFERED_H */ -/**** ended inlining zstd_v06.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) -/**** start inlining zstd_v07.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv07_H_235446 -#define ZSTDv07_H_235446 - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIBv07_API __declspec(dllexport) -#else -# define ZSTDLIBv07_API -#endif - - -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv07_getDecompressedSize() : -* @return : decompressed size if known, 0 otherwise. - note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. - note 2 : decompressed size could be wrong or intentionally modified ! - always ensure results fit within application's authorized limits */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv07_decompress() : - `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/** -ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/*====== Helper functions ======*/ -ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ - - -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv07_decompressDCtx() : -* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv07_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************** -* Advanced Dictionary API -****************************/ -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); - -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv07_frameParams; - -ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ - - - - -/* ************************************* -* Streaming functions -***************************************/ -typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); - -ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void); - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv07_H_235446 */ -/**** ended inlining zstd_v07.h ****/ -#endif - -/** ZSTD_isLegacy() : - @return : > 0 if supported by legacy decoder. 0 otherwise. - return value is the version. -*/ -MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) -{ - U32 magicNumberLE; - if (srcSize<4) return 0; - magicNumberLE = MEM_readLE32(src); - switch(magicNumberLE) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case ZSTDv01_magicNumberLE:return 1; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case ZSTDv02_magicNumber : return 2; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case ZSTDv03_magicNumber : return 3; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case ZSTDv04_magicNumber : return 4; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case ZSTDv05_MAGICNUMBER : return 5; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case ZSTDv06_MAGICNUMBER : return 6; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case ZSTDv07_MAGICNUMBER : return 7; -#endif - default : return 0; - } -} - - -MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) -{ - U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ -#if (ZSTD_LEGACY_SUPPORT <= 5) - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } -#endif - return 0; /* should not be possible */ -} - - -MEM_STATIC size_t ZSTD_decompressLegacy( - void* dst, size_t dstCapacity, - const void* src, size_t compressedSize, - const void* dict,size_t dictSize) -{ - U32 const version = ZSTD_isLegacy(src, compressedSize); - (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { size_t result; - ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv05_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { size_t result; - ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv06_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { size_t result; - ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv07_freeDCtx(zd); - return result; - } -#endif - default : - return ERROR(prefix_unknown); - } -} - -MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo; - U32 const version = ZSTD_isLegacy(src, srcSize); - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - ZSTDv01_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - ZSTDv02_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - ZSTDv03_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - ZSTDv04_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - ZSTDv05_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - ZSTDv06_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - ZSTDv07_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif - default : - frameSizeInfo.compressedSize = ERROR(prefix_unknown); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - break; - } - if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) { - frameSizeInfo.compressedSize = ERROR(srcSize_wrong); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - } - return frameSizeInfo; -} - -MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize); - return frameSizeInfo.compressedSize; -} - -MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) -{ - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); -#endif - } -} - - -MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, - const void* dict, size_t dictSize) -{ - DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion); - if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); - switch(newVersion) - { - default : - case 1 : - case 2 : - case 3 : - (void)dict; (void)dictSize; - return 0; -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv04_decompressInit(dctx); - ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif - } -} - - - -MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, - ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version); - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; (void)output; (void)input; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif - } -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_LEGACY_H */ -/**** ended inlining ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -29243,7 +38434,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); @@ -29349,12 +38540,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); + return ddict->dictID; } /**** ended inlining decompress/zstd_ddict.c ****/ /**** start inlining decompress/zstd_decompress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29411,20 +38602,20 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) * Dependencies *********************************************************/ /**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ +/**** skipping file: ../common/bits.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** start inlining zstd_decompress_block.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29458,6 +38649,12 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + /* ZSTD_decompressBlock_internal() : * decompress block, starting at `src`, * into destination buffer `dst`. @@ -29466,7 +38663,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame); + const void* src, size_t srcSize, const streaming_operation streaming); /* ZSTD_buildFSETable() : * generate FSE decoding table for one symbol (ll, ml or off) @@ -29479,16 +38676,21 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ /**** ended inlining zstd_decompress_block.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** skipping file: ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -29498,11 +38700,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, *************************************/ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 -#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. - * Currently, that means a 0.75 load factor. - * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded - * the load factor of the ddict hash set. - */ +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ #define DDICT_HASHSET_TABLE_BASE_SIZE 64 #define DDICT_HASHSET_RESIZE_FACTOR 2 @@ -29596,12 +38798,15 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); - ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; - ret->ddictPtrCount = 0; - if (!ret || !ret->ddictPtrTable) { + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); return NULL; } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; return ret; } @@ -29660,6 +38865,8 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) @@ -29674,11 +38881,16 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; +#endif dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; - dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION @@ -29699,8 +38911,7 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) return dctx; } -ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); @@ -29711,10 +38922,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) } } +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + ZSTD_DCtx* ZSTD_createDCtx(void) { DEBUGLOG(3, "ZSTD_createDCtx"); - return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } static void ZSTD_clearDict(ZSTD_DCtx* dctx) @@ -29799,6 +39015,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) return 0; } +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -29834,16 +39063,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -29851,8 +39104,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; + zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START; + zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE; + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); return 0; } RETURN_ERROR(prefix_unknown, ""); @@ -29885,7 +39140,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : dictID = ip[pos]; pos++; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; @@ -29893,7 +39150,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(fcsID) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; case 2 : frameContentSize = MEM_readLE32(ip+pos); break; @@ -29917,12 +39176,11 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize) { return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); } - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -29936,7 +39194,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; } #endif - { ZSTD_frameHeader zfh; + { ZSTD_FrameHeader zfh; if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (zfh.frameType == ZSTD_skippableFrame) { @@ -29956,18 +39214,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported, ""); - { - size_t const skippableSize = skippableHeaderSize + sizeU32; + { size_t const skippableSize = skippableHeaderSize + sizeU32; RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + /** ZSTD_findDecompressedSize() : - * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or * skippable frames - * @return : decompressed size of the frames contained */ + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; @@ -29977,9 +39269,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -29987,17 +39277,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) continue; } - { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; } + /* skip to next frame */ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); src = (const BYTE *)src + frameSrcSize; srcSize -= frameSrcSize; @@ -30061,17 +39351,17 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) return frameSizeInfo; } -static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) { ZSTD_frameSizeInfo frameSizeInfo; ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameSizeInfoLegacy(src, srcSize); #endif - if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); assert(ZSTD_isError(frameSizeInfo.compressedSize) || @@ -30082,10 +39372,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize const BYTE* const ipstart = ip; size_t remainingSize = srcSize; size_t nbBlocks = 0; - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; /* Extract Frame Header */ - { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); if (ZSTD_isError(ret)) return ZSTD_errorFrameSizeInfo(ret); if (ret > 0) @@ -30119,28 +39409,31 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } + frameSizeInfo.nbBlocks = nbBlocks; frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + /** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ + * See docs in zstd.h + * Note: compatible with legacy mode */ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); - return frameSizeInfo.compressedSize; + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); } /** ZSTD_decompressBound() : * compatible with legacy mode - * `src` must point to the start of a ZSTD frame or a skippeable frame + * `src` must point to the start of a ZSTD frame or a skippable frame * `srcSize` must be at least as large as the frame contained * @return : the maximum decompressed size of the compressed source */ @@ -30149,7 +39442,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long bound = 0; /* Iterate over each frame */ while (srcSize > 0) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); size_t const compressedSize = frameSizeInfo.compressedSize; unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) @@ -30162,6 +39455,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) return bound; } +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_FrameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} /*-************************************************************* * Frame decoding @@ -30187,7 +39522,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -30204,10 +39539,10 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, return regenSize; } -static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming) { #if ZSTD_TRACE - if (dctx->traceCtx) { + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); trace.version = ZSTD_VERSION_NUMBER; @@ -30263,10 +39598,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; + memset(&blockProperties, 0, sizeof(blockProperties)); // shut up gcc warning size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); if (ZSTD_isError(cBlockSize)) return cBlockSize; @@ -30274,27 +39615,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: RETURN_ERROR(corruption_detected, "invalid block type"); } - - if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->validateChecksum) + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { XXH64_update(&dctx->xxhState, op, decodedSize); - if (decodedSize != 0) + } + if (decodedSize) /* support dst = NULL,0 */ { op += decodedSize; + } assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; @@ -30318,12 +39680,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); } -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, @@ -30343,7 +39708,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) { + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { size_t decodedSize; size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); if (ZSTD_isError(frameSize)) return frameSize; @@ -30353,6 +39718,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + assert(decodedSize <= dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -30364,17 +39738,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, } #endif - { U32 const magicNumber = MEM_readLE32(src); - DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; - continue; + continue; /* check next frame */ } } if (ddict) { @@ -30428,7 +39803,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) switch (dctx->dictUses) { default: assert(0 /* Impossible */); - /* fall-through */ + ZSTD_FALLTHROUGH; case ZSTD_dont_use: ZSTD_clearDict(dctx); return NULL; @@ -30450,7 +39825,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); @@ -30470,8 +39845,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -30484,7 +39859,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t return dctx->expected; if (dctx->bType != bt_raw) return dctx->expected; - return MIN(MAX(inputSize, 1), dctx->expected); + return BOUNDED(1, inputSize, dctx->expected); } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -30492,7 +39867,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { { default: /* should not happen */ assert(0); + ZSTD_FALLTHROUGH; case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: @@ -30504,6 +39881,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { case ZSTDds_checkChecksum: return ZSTDnit_checksum; case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; } @@ -30587,7 +39965,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : @@ -30656,6 +40035,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; @@ -30668,7 +40048,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -30709,11 +40089,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -30802,7 +40182,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { assert(dctx != NULL); #if ZSTD_TRACE - dctx->traceCtx = ZSTD_trace_decompress_begin(dctx); + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; #endif dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ dctx->stage = ZSTDds_getFrameHeaderSize; @@ -30812,10 +40192,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -30874,7 +40255,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -30883,7 +40264,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; @@ -30912,7 +40293,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, ZSTD_DStream* ZSTD_createDStream(void) { DEBUGLOG(3, "ZSTD_createDStream"); - return ZSTD_createDStream_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) @@ -30922,7 +40303,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) { - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_internal(customMem); } size_t ZSTD_freeDStream(ZSTD_DStream* zds) @@ -30990,7 +40371,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -30998,6 +40381,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -31008,6 +40392,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } @@ -31079,6 +40464,15 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + default:; } bounds.error = ERROR(parameter_unsupported); @@ -31119,6 +40513,12 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value case ZSTD_d_refMultipleDDicts: *value = (int)dctx->refMultipleDDicts; return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -31152,6 +40552,14 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value } dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -31163,6 +40571,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { dctx->streamStage = zdss_init; dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { @@ -31179,10 +40588,17 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) return ZSTD_sizeof_DCtx(dctx); } -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) { - size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); - unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, @@ -31190,6 +40606,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long return minRBSize; } +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + size_t ZSTD_estimateDStreamSize(size_t windowSize) { size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); @@ -31201,7 +40622,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize) size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) { U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; RETURN_ERROR_IF(err>0, srcSize_wrong, ""); @@ -31296,6 +40717,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB U32 someMoreWork = 1; DEBUGLOG(5, "ZSTD_decompressStream"); + assert(zds != NULL); RETURN_ERROR_IF( input->pos > input->size, srcSize_wrong, @@ -31316,10 +40738,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(5, "stage zdss_init => transparent reset "); zds->streamStage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) zds->legacyVersion = 0; +#endif zds->hostageByte = 0; zds->expectedOutBuffer = *output; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); @@ -31336,7 +40760,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); @@ -31368,6 +40791,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -31379,14 +40807,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; - DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); ip = istart + cSize; - op += decompressedSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ zds->expected = 0; zds->streamStage = zdss_init; someMoreWork = 0; @@ -31405,7 +40834,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); - if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -31421,11 +40851,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered - ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) : 0; ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); @@ -31457,7 +40889,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outBuffSize = neededOutBuffSize; } } } zds->streamStage = zdss_read; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_read: DEBUGLOG(5, "stage zdss_read"); @@ -31470,13 +40902,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; } } if (ip==iend) { someMoreWork = 0; break; } /* no more input */ zds->streamStage = zdss_load; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); @@ -31484,7 +40917,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -31493,8 +40926,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -31504,14 +40940,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -31525,7 +40964,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -31538,8 +40977,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -31576,18 +41015,24 @@ size_t ZSTD_decompressStream_simpleArgs ( void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } /**** ended inlining decompress/zstd_decompress.c ****/ /**** start inlining decompress/zstd_decompress_block.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -31608,12 +41053,12 @@ size_t ZSTD_decompressStream_simpleArgs ( /**** skipping file: ../common/mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** skipping file: zstd_decompress_block.h ****/ +/**** skipping file: ../common/bits.h ****/ /*_******************************************************* * Macros @@ -31639,6 +41084,13 @@ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } * Block decoding ***************************************************************/ +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, @@ -31657,36 +41109,90 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, } } +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} -/* Hidden declaration for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); /*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * * @return : nb of bytes read from src (< srcSize ) * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); switch(litEncType) { case set_repeat: DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); - /* fall-through */ + ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31709,8 +41215,15 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -31719,13 +41232,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -31733,20 +41247,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); } } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); @@ -31754,13 +41277,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case set_basic: { size_t litSize, lhSize; U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31773,27 +41296,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; return lhSize+litSize; } case set_rle: { U32 const lhlCode = ((istart[0]) >> 2) & 3; size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -31802,16 +41340,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -31822,6 +41372,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions @@ -31829,7 +41391,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ + * - prettify output, report below, test with fuzzer to ensure it's correct */ /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<nbBits = 0; cell->nextState = 0; assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->nbAdditionalBits = nbAddBits; cell->baseValue = baseValue; } @@ -31955,7 +41517,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB FORCE_INLINE_TEMPLATE void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; @@ -32018,14 +41580,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -32052,7 +41615,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -32063,10 +41626,10 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; u max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); } *DTablePtr = DTableSpace; @@ -32176,11 +41739,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* SeqHead */ nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); - return 1; - } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); @@ -32193,11 +41751,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, } *nbSeqPtr = nbSeq; + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + /* FSE table descriptors */ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6); + SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3); + SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3); ip++; /* Build DTables */ @@ -32208,7 +41774,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -32220,7 +41786,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -32232,7 +41798,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } @@ -32246,7 +41812,6 @@ typedef struct { size_t litLength; size_t matchLength; size_t offset; - const BYTE* match; } seq_t; typedef struct { @@ -32260,9 +41825,6 @@ typedef struct { ZSTD_fseState stateOffb; ZSTD_fseState stateML; size_t prevOffset[ZSTD_REP_NUM]; - const BYTE* prefixStart; - const BYTE* dictEnd; - size_t pos; } seqState_t; /*! ZSTD_overlapCopy8() : @@ -32305,7 +41867,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * The src buffer must be before the dst buffer. */ -static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t const diff = op - ip; BYTE* const oend = op + length; @@ -32321,6 +41883,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ assert(length >= 8); ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; assert(op - ip >= 8); assert(op <= oend); } @@ -32335,12 +41898,35 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ assert(oend > oend_w); ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ip += oend_w - op; - op = oend_w; + op += oend_w - op; } /* Handle the leftovers. */ while (op < oend) *op++ = *ip++; } +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + /* ZSTD_execSequenceEnd(): * This version handles cases that are near the end of the output buffer. It requires * more careful checks to make sure there is no overflow. By separating out these hard @@ -32350,10 +41936,11 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). */ FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -32376,27 +41963,78 @@ size_t ZSTD_execSequenceEnd(BYTE* op, if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart-match); + match = dictEnd - (prefixStart - match); if (match + sequence.matchLength <= dictEnd) { ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -32407,6 +42045,104 @@ size_t ZSTD_execSequence(BYTE* op, assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); /* Handle edge cases in a slow path: * - Read beyond end of literals * - Match end is within WILDCOPY_OVERLIMIT of oend @@ -32416,7 +42152,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -32484,6 +42220,7 @@ size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } + static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) { @@ -32497,24 +42234,14 @@ ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqS } FORCE_INLINE_TEMPLATE void -ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) { - ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) -{ - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; + DStatePtr->state = nextState + lowBits; } /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 * bits before reloading. This value is the maximum number of bytes we read * after reloading when we are decoding long offsets. */ @@ -32524,123 +42251,136 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD : 0) typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; -typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) { seq_t seq; - ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; - ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; - ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; - U32 const llBase = llDInfo.baseValue; - U32 const mlBase = mlDInfo.baseValue; - U32 const ofBase = ofDInfo.baseValue; - BYTE const llBits = llDInfo.nbAdditionalBits; - BYTE const mlBits = mlDInfo.nbAdditionalBits; - BYTE const ofBits = ofDInfo.nbAdditionalBits; - BYTE const totalBits = llBits+mlBits+ofBits; - - /* sequence */ - { size_t offset; - if (ofBits > 1) { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } else { - U32 const ll0 = (llBase == 0); - if (LIKELY((ofBits == 0))) { - if (LIKELY(!ll0)) - offset = seqState->prevOffset[0]; - else { - offset = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } - seq.offset = offset; - } - - seq.matchLength = mlBase; - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - seq.litLength = llBase; - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - - if (prefetch == ZSTD_p_prefetch) { - size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, offset is only used for prefetching */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update - * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). - * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). - * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the - * better option, so it is the default for other compilers. But, if you - * measure that it is worse, please put up a pull request. + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. */ - { -#if defined(__GNUC__) && !defined(__clang__) - const int kUseUpdateFseState = 1; +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); #else - const int kUseUpdateFseState = 0; + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; #endif - if (kUseUpdateFseState) { - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - } else { - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); } } return seq; } -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) { size_t const windowSize = dctx->fParams.windowSize; /* No dictionary used. */ @@ -32654,30 +42394,33 @@ MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefix /* Dictionary is active. */ return 1; } +#endif -MEM_STATIC void ZSTD_assertValidSequence( +static void ZSTD_assertValidSequence( ZSTD_DCtx const* dctx, BYTE const* op, BYTE const* oend, seq_t const seq, BYTE const* prefixStart, BYTE const* virtualStart) { #if DEBUGLEVEL >= 1 - size_t const windowSize = dctx->fParams.windowSize; - size_t const sequenceSize = seq.litLength + seq.matchLength; - BYTE const* const oLitEnd = op + seq.litLength; - DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - assert(op <= oend); - assert((size_t)(oend - op) >= sequenceSize); - assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); - if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { - size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); - /* Offset must be within the dictionary. */ - assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); - assert(seq.offset <= windowSize + dictSize); - } else { - /* Offset must be within our window. */ - assert(seq.offset <= windowSize); + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } } #else (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; @@ -32686,31 +42429,30 @@ MEM_STATIC void ZSTD_assertValidSequence( #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - (void)frame; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - /* Regen sequences */ + /* Literals are split between internal buffer & output buffer */ if (nbSeq) { seqState_t seqState; - size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } RETURN_ERROR_IF( @@ -32726,134 +42468,331 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ #if defined(__GNUC__) && defined(__x86_64__) - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * I've been able to reproduce this issue on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 4"); + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif #endif - for ( ; ; ) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); #endif - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - BIT_reloadDStream(&(seqState.DStream)); - op += oneSeqSize; - /* gcc and clang both don't like early returns in this loop. - * Instead break and check for an error at the end of the loop. - */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) { - error = oneSeqSize; - break; + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } - if (UNLIKELY(!--nbSeq)) break; } /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (ZSTD_isError(error)) return error; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } - return op-ostart; + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - (void)frame; /* Regen sequences */ if (nbSeq) { -#define STORED_SEQS 4 +#define STORED_SEQS 8 #define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 +#define ADVANCED_SEQS STORED_SEQS seq_t sequences[STORED_SEQS]; int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); seqState_t seqState; int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + dctx->fseEntropy = 1; { int i; for (i=0; ientropy.rep[i]; } - seqState.prefixStart = prefixStart; - seqState.pos = (size_t)(op-prefixStart); - seqState.dictEnd = dictEnd; assert(dst != NULL); assert(iend >= ip); RETURN_ERROR_IF( @@ -32864,37 +42803,95 @@ ZSTD_decompressSequencesLong_body( ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } - RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } /* save reps for next block */ @@ -32902,25 +42899,34 @@ ZSTD_decompressSequencesLong_body( } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); + ZSTD_memmove(op, litPtr, lastLLSize); op += lastLLSize; } } - return op-ostart; + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -32929,53 +42935,65 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #if DYNAMIC_BMI2 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ #endif /* DYNAMIC_BMI2 */ -typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame); - #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -32990,69 +43008,114 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) total += 1; + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ - - return total; + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) + const void* src, size_t srcSize, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -33060,6 +43123,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. @@ -33067,6 +43147,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -33074,37 +43159,55 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif dctx->ddictIsCold = 0; #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif } } +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ @@ -33116,21 +43219,32 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + dctx->isFrameDecompression = 0; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t dSize; - ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); } /**** ended inlining decompress/zstd_decompress_block.c ****/ /**** start inlining dictBuilder/cover.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -33152,39 +43266,32 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, /*-************************************* * Dependencies ***************************************/ +/* qsort_r is an extension. */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include /* fprintf */ -#include /* malloc, free, qsort */ +#include /* malloc, free, qsort_r */ + #include /* memset */ #include /* clock */ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/pool.h ****/ -/**** skipping file: ../common/threading.h ****/ -/**** start inlining cover.h ****/ -/* - * Copyright (c) 2017-2021, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif -#include /* fprintf */ -#include /* malloc, free, qsort */ -#include /* memset */ -#include /* clock */ /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/pool.h ****/ /**** skipping file: ../common/threading.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** start inlining zdict.h ****/ +/**** skipping file: ../common/bits.h ****/ +/**** start inlining ../zdict.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -33193,34 +43300,184 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, * You may select, at your option, one of the above-listed licenses. */ -#ifndef DICTBUILDER_H_001 -#define DICTBUILDER_H_001 - -#if defined (__cplusplus) -extern "C" { -#endif +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H /*====== Dependencies ======*/ #include /* size_t */ +#if defined (__cplusplus) +extern "C" { +#endif /* ===== ZDICTLIB_API : control library symbols visibility ===== */ -#ifndef ZDICTLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE # endif #endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZDICTLIB_API ZDICTLIB_VISIBILITY + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif #endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBLE +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + /*! ZDICT_trainFromBuffer(): * Train a dictionary from an array of samples. @@ -33247,9 +43504,16 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap const size_t* samplesSizes, unsigned nbSamples); typedef struct { - int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ - unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */ + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ } ZDICT_params_t; /*! ZDICT_finalizeDictionary(): @@ -33276,8 +43540,7 @@ typedef struct { * is presumed that the most profitable content is at the end of the dictionary, * since that is the cheapest to reference. * - * `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes. - * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN). * * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), * or an error code, which can be tested by ZDICT_isError(). @@ -33300,9 +43563,29 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); +#if defined (__cplusplus) +} +#endif +#endif /* ZSTD_ZDICT_H */ -#ifdef ZDICT_STATIC_LINKING_ONLY +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC + +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif /* ==================================================================================== * The definitions in this section are considered experimental. @@ -33311,8 +43594,9 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); * Use them only in association with static linking. * ==================================================================================== */ -#define ZDICT_CONTENTSIZE_MIN 128 #define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 /*! ZDICT_cover_params_t: * k and d are the only required parameters. @@ -33357,7 +43641,7 @@ typedef struct { * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_cover_params_t parameters); @@ -33379,7 +43663,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters); @@ -33400,7 +43684,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters); @@ -33423,7 +43707,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t* parameters); @@ -33448,7 +43732,7 @@ typedef struct { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_legacy_params_t parameters); @@ -33460,36 +43744,52 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( or _CRT_SECURE_NO_WARNINGS in Visual. Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS -# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ #else # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") -# define ZDICT_DEPRECATED(message) ZDICTLIB_API +# define ZDICT_DEPRECATED(message) # endif #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); - -#endif /* ZDICT_STATIC_LINKING_ONLY */ - #if defined (__cplusplus) } #endif -#endif /* DICTBUILDER_H_001 */ -/**** ended inlining zdict.h ****/ +#endif /* ZSTD_ZDICT_H_STATIC */ +/**** ended inlining ../zdict.h ****/ +/**** start inlining cover.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + +/**** skipping file: ../common/threading.h ****/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../zdict.h ****/ /** * COVER_best_t is used for two purposes: @@ -33626,15 +43926,17 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection); size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize); /**** ended inlining cover.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** skipping file: zdict.h ****/ /*-************************************* * Constants ***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define COVER_DEFAULT_SPLITPOINT 1.0 @@ -33642,7 +43944,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection); * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -33665,7 +43967,7 @@ static clock_t g_time = 0; #undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ - if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ + if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ } \ @@ -33819,8 +44121,10 @@ typedef struct { unsigned d; } COVER_ctx_t; -/* We need a global context for qsort... */ +#if !defined(_GNU_SOURCE) && !defined(__APPLE__) && !defined(_MSC_VER) +/* C90 only offers qsort() that needs a global context. */ static COVER_ctx_t *g_coverCtx = NULL; +#endif /*-************************************* * Helper functions @@ -33863,11 +44167,15 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { /** * Same as COVER_cmp() except ties are broken by pointer value - * NOTE: g_coverCtx must be set to call this function. A global is required because - * qsort doesn't take an opaque pointer. */ -static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { - int result = COVER_cmp(g_coverCtx, lp, rp); +#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) { +#elif defined(_GNU_SOURCE) +static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) { +#else /* C90 fallback.*/ +static int COVER_strict_cmp(const void *lp, const void *rp) { +#endif + int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } @@ -33876,21 +44184,58 @@ static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) { /** * Faster version for d <= 8. */ -static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) { - int result = COVER_cmp8(g_coverCtx, lp, rp); +#if (defined(_WIN32) && defined(_MSC_VER)) || defined(__APPLE__) +static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) { +#elif defined(_GNU_SOURCE) +static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) { +#else /* C90 fallback.*/ +static int COVER_strict_cmp8(const void *lp, const void *rp) { +#endif + int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp); if (result == 0) { result = lp < rp ? -1 : 1; } return result; } +/** + * Abstract away divergence of qsort_r() parameters. + * Hopefully when C11 become the norm, we will be able + * to clean it up. + */ +static void stableSort(COVER_ctx_t *ctx) { +#if defined(__APPLE__) + qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), + ctx, + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#elif defined(_GNU_SOURCE) + qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), + ctx); +#elif defined(_WIN32) && defined(_MSC_VER) + qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp), + ctx); +#elif defined(__OpenBSD__) + g_coverCtx = ctx; + mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#else /* C90 fallback.*/ + g_coverCtx = ctx; + /* TODO(cavalcanti): implement a reentrant qsort() when is not available. */ + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); +#endif +} + /** * Returns the first pointer in [first, last) whose element does not compare * less than value. If no such element exists it returns last. */ -static const size_t *COVER_lower_bound(const size_t *first, const size_t *last, +static const size_t *COVER_lower_bound(const size_t* first, const size_t* last, size_t value) { - size_t count = last - first; + size_t count = (size_t)(last - first); + assert(last >= first); while (count != 0) { size_t step = count / 2; const size_t *ptr = first; @@ -34129,14 +44474,15 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) { /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `COVER_ctx_destroy()`. */ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, - unsigned d, double splitPoint) { + unsigned d, double splitPoint) +{ const BYTE *const samples = (const BYTE *)samplesBuffer; const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); /* Split samples into testing and training sets */ @@ -34205,17 +44551,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, for (i = 0; i < ctx->suffixSize; ++i) { ctx->suffix[i] = i; } - /* qsort doesn't take an opaque pointer, so pass as a global. - * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is. - */ - g_coverCtx = ctx; -#if defined(__OpenBSD__) - mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32), - (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); -#else - qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), - (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); -#endif + stableSort(ctx); } DISPLAYLEVEL(2, "Computing frequencies\n"); /* For each dmer group (group of positions with the same first d bytes): @@ -34234,7 +44570,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel) { - const double ratio = (double)nbDmers / maxDictSize; + const double ratio = (double)nbDmers / (double)maxDictSize; if (ratio >= 10) { return; } @@ -34320,7 +44656,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, return tail; } -ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_cover_params_t parameters) @@ -34330,7 +44666,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( COVER_map_t activeDmers; parameters.splitPoint = 1.0; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Checks */ if (!COVER_checkParameters(parameters, dictBufferCapacity)) { DISPLAYLEVEL(1, "Cover parameters incorrect\n"); @@ -34494,8 +44830,10 @@ void COVER_best_start(COVER_best_t *best) { * Decrements liveJobs and signals any waiting threads if liveJobs == 0. * If this dictionary is the best so far save it and its parameters. */ -void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, - COVER_dictSelection_t selection) { +void COVER_best_finish(COVER_best_t* best, + ZDICT_cover_params_t parameters, + COVER_dictSelection_t selection) +{ void* dict = selection.dictContent; size_t compressedSize = selection.totalCompressedSize; size_t dictSize = selection.dictSize; @@ -34538,9 +44876,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, } } +static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz) +{ + COVER_dictSelection_t ds; + ds.dictContent = buf; + ds.dictSize = s; + ds.totalCompressedSize = csz; + return ds; +} + COVER_dictSelection_t COVER_dictSelectionError(size_t error) { - COVER_dictSelection_t selection = { NULL, 0, error }; - return selection; + return setDictSelection(NULL, 0, error); } unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) { @@ -34559,8 +44905,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe size_t largestCompressed = 0; BYTE* customDictContentEnd = customDictContent + dictContentSize; - BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity); - BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity); + BYTE* largestDictbuffer = (BYTE*)malloc(dictBufferCapacity); + BYTE* candidateDictBuffer = (BYTE*)malloc(dictBufferCapacity); double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00; if (!largestDictbuffer || !candidateDictBuffer) { @@ -34593,9 +44939,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe } if (params.shrinkDict == 0) { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; free(candidateDictBuffer); - return selection; + return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize); } largestDict = dictContentSize; @@ -34627,20 +44972,16 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe return COVER_dictSelectionError(totalCompressedSize); } - if (totalCompressedSize <= largestCompressed * regressionTolerance) { - COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize }; + if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) { free(largestDictbuffer); - return selection; + return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize ); } dictContentSize *= 2; } dictContentSize = largestDict; totalCompressedSize = largestCompressed; - { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; - free(candidateDictBuffer); - return selection; - } + free(candidateDictBuffer); + return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize ); } /** @@ -34703,7 +45044,7 @@ _cleanup: free(freqs); } -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters) @@ -34913,11 +45254,6 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( #ifndef _DIVSUFSORT_H #define _DIVSUFSORT_H 1 -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - - /*- Prototypes -*/ /** @@ -34945,11 +45281,6 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP); int divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - #endif /* _DIVSUFSORT_H */ /**** ended inlining divsufsort.h ****/ @@ -36825,7 +47156,7 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * /**** ended inlining dictBuilder/divsufsort.c ****/ /**** start inlining dictBuilder/fastcover.c ****/ /* - * Copyright (c) 2018-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -36842,21 +47173,29 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * #include /* memset */ #include /* clock */ +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif + /**** skipping file: ../common/mem.h ****/ /**** skipping file: ../common/pool.h ****/ /**** skipping file: ../common/threading.h ****/ -/**** skipping file: cover.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../compress/zstd_compress_internal.h ****/ -#ifndef ZDICT_STATIC_LINKING_ONLY -#define ZDICT_STATIC_LINKING_ONLY -#endif -/**** skipping file: zdict.h ****/ +/**** skipping file: ../zdict.h ****/ +/**** skipping file: cover.h ****/ /*-************************************* * Constants ***************************************/ +/** +* There are 32bit indexes used to ref samples, so limit samples size to 4GB +* on 64bit builds. +* For 32bit builds we choose 1 GB. +* Most 32bit platforms have 2GB user-mode addressable space and we allocate a large +* contiguous buffer, so 1GB is already a high limit. +*/ #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define FASTCOVER_MAX_F 31 #define FASTCOVER_MAX_ACCEL 10 @@ -36869,7 +47208,7 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -37122,7 +47461,7 @@ FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx) /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `FASTCOVER_ctx_destroy()`. @@ -37363,7 +47702,7 @@ FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, } -ZDICTLIB_API size_t +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, @@ -37374,7 +47713,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, ZDICT_cover_params_t coverParams; FASTCOVER_accel_t accelParams; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Assign splitPoint and f if not provided */ parameters.splitPoint = 1.0; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; @@ -37432,7 +47771,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, } -ZDICTLIB_API size_t +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, @@ -37457,7 +47796,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; const unsigned shrinkDict = 0; /* Local variables */ - const int displayLevel = parameters->zParams.notificationLevel; + const int displayLevel = (int)parameters->zParams.notificationLevel; unsigned iteration = 1; unsigned d; unsigned k; @@ -37541,7 +47880,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( data->parameters.splitPoint = splitPoint; data->parameters.steps = kSteps; data->parameters.shrinkDict = shrinkDict; - data->parameters.zParams.notificationLevel = g_displayLevel; + data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; /* Check the parameters */ if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) { @@ -37585,7 +47924,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( /**** ended inlining dictBuilder/fastcover.c ****/ /**** start inlining dictBuilder/zdict.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -37627,18 +47966,19 @@ ZDICT_optimizeTrainFromBuffer_fastCover( #include /* fprintf, fopen, ftello64 */ #include /* clock */ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -/**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: divsufsort.h ****/ #ifndef ZDICT_STATIC_LINKING_ONLY # define ZDICT_STATIC_LINKING_ONLY #endif -/**** skipping file: zdict.h ****/ + +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/xxhash.h ****/ /**** skipping file: ../compress/zstd_compress_internal.h ****/ +/**** skipping file: ../zdict.h ****/ +/**** skipping file: divsufsort.h ****/ +/**** skipping file: ../common/bits.h ****/ /*-************************************* @@ -37659,9 +47999,9 @@ static const U32 g_selectivity_default = 9; * Console display ***************************************/ #undef DISPLAY -#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#define DISPLAY(...) do { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } while (0) #undef DISPLAYLEVEL -#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) do { if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } } while (0) /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } @@ -37715,65 +48055,6 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize) /*-******************************************************** * Dictionary training functions **********************************************************/ -static unsigned ZDICT_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r=0; - _BitScanForward( &r, (U32)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } } -} - - /*! ZDICT_count() : Count the nb of common bytes between 2 pointers. Note : this function presumes end of buffer followed by noisy guard band. @@ -37788,7 +48069,7 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch) pMatch = (const char*)pMatch+sizeof(size_t); continue; } - pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + pIn = (const char*)pIn+ZSTD_NbCommonBytes(diff); return (size_t)((const char*)pIn - pStart); } } @@ -37820,7 +48101,7 @@ static dictItem ZDICT_analyzePos( U32 savings[LLIMIT] = {0}; const BYTE* b = (const BYTE*)buffer; size_t maxLength = LLIMIT; - size_t pos = suffix[start]; + size_t pos = (size_t)suffix[start]; U32 end = start; dictItem solution; @@ -37954,7 +48235,7 @@ static dictItem ZDICT_analyzePos( savings[i] = savings[i-1] + (lengthList[i] * (i-3)); DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", - (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength); + (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength); solution.pos = (U32)pos; solution.length = (U32)maxLength; @@ -37964,7 +48245,7 @@ static dictItem ZDICT_analyzePos( { U32 id; for (id=start; id1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -38027,7 +48308,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -38121,10 +48402,16 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; # undef DISPLAYUPDATE -# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ - if (ZDICT_clockSpan(displayClock) > refreshRate) \ - { displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (notificationLevel>=4) fflush(stderr); } } +# define DISPLAYUPDATE(l, ...) \ + do { \ + if (notificationLevel>=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) { \ + displayClock = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + if (notificationLevel>=4) fflush(stderr); \ + } \ + } while (0) /* init */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ @@ -38167,7 +48454,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, if (solution.length==0) { cursor++; continue; } ZDICT_insertDictItem(dictList, dictListSize, solution, buffer); cursor += solution.length; - DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); + DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0); } } _cleanup: @@ -38210,15 +48497,15 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); + { size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } } - cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; } if (cSize) { /* if == 0; block is not compressible */ - const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc); + const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc); /* literals stats */ { const BYTE* bytePtr; @@ -38246,9 +48533,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, } if (nbSeq >= 2) { /* rep offsets */ - const seqDef* const seq = seqStorePtr->sequencesStart; - U32 offset1 = seq[0].offset - 3; - U32 offset2 = seq[1].offset - 3; + const SeqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offBase - ZSTD_REP_NUM; + U32 offset2 = seq[1].offBase - ZSTD_REP_NUM; if (offset1 >= MAXREPOFFSET) offset1 = 0; if (offset2 >= MAXREPOFFSET) offset2 = 0; repOffsets[offset1] += 3; @@ -38319,6 +48606,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); BYTE* dstPtr = (BYTE*)dstBuffer; + U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32]; /* init */ DEBUGLOG(4, "ZDICT_analyzeEntropy"); @@ -38351,8 +48639,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, pos += fileSizes[u]; } + if (notificationLevel >= 4) { + /* writeStats */ + DISPLAYLEVEL(4, "Offset Code Frequencies : \n"); + for (u=0; u<=offcodeMax; u++) { + DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]); + } } + /* analyze, build stats, starting with literals */ - { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + { size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(maxNbBits)) { eSize = maxNbBits; DISPLAYLEVEL(1, " HUF_buildCTable error \n"); @@ -38361,7 +48656,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */ DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n"); ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */ - maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); assert(maxNbBits==9); } huffLog = (U32)maxNbBits; @@ -38402,7 +48697,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, llLog = (U32)errorCode; /* write result to buffer */ - { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); + { size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(hhSize)) { eSize = hhSize; DISPLAYLEVEL(1, "HUF_writeCTable error \n"); @@ -38457,7 +48752,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); #else /* at this stage, we don't use the result of "most common first offset", - as the impact of statistics is not properly evaluated */ + * as the impact of statistics is not properly evaluated */ MEM_writeLE32(dstPtr+0, repStartValue[0]); MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); @@ -38473,6 +48768,17 @@ _cleanup: } +/** + * @returns the maximum repcode value + */ +static U32 ZDICT_maxRep(U32 const reps[ZSTD_REP_NUM]) +{ + U32 maxRep = reps[0]; + int r; + for (r = 1; r < ZSTD_REP_NUM; ++r) + maxRep = MAX(maxRep, reps[r]); + return maxRep; +} size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, const void* customDictContent, size_t dictContentSize, @@ -38484,11 +48790,13 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, BYTE header[HBUFFSIZE]; int const compressionLevel = (params.compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; + /* The final dictionary content must be at least as large as the largest repcode */ + size_t const minContentSize = (size_t)ZDICT_maxRep(repStartValue); + size_t paddingSize; /* check conditions */ DEBUGLOG(4, "ZDICT_finalizeDictionary"); if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall); - if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong); if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall); /* dictionary header */ @@ -38512,12 +48820,43 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, hSize += eSize; } - /* copy elements in final buffer ; note : src and dst buffer can overlap */ - if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize; - { size_t const dictSize = hSize + dictContentSize; - char* dictEnd = (char*)dictBuffer + dictSize; - memmove(dictEnd - dictContentSize, customDictContent, dictContentSize); - memcpy(dictBuffer, header, hSize); + /* Shrink the content size if it doesn't fit in the buffer */ + if (hSize + dictContentSize > dictBufferCapacity) { + dictContentSize = dictBufferCapacity - hSize; + } + + /* Pad the dictionary content with zeros if it is too small */ + if (dictContentSize < minContentSize) { + RETURN_ERROR_IF(hSize + minContentSize > dictBufferCapacity, dstSize_tooSmall, + "dictBufferCapacity too small to fit max repcode"); + paddingSize = minContentSize - dictContentSize; + } else { + paddingSize = 0; + } + + { + size_t const dictSize = hSize + paddingSize + dictContentSize; + + /* The dictionary consists of the header, optional padding, and the content. + * The padding comes before the content because the "best" position in the + * dictionary is the last byte. + */ + BYTE* const outDictHeader = (BYTE*)dictBuffer; + BYTE* const outDictPadding = outDictHeader + hSize; + BYTE* const outDictContent = outDictPadding + paddingSize; + + assert(dictSize <= dictBufferCapacity); + assert(outDictContent + dictContentSize == (BYTE*)dictBuffer + dictSize); + + /* First copy the customDictContent into its final location. + * `customDictContent` and `dictBuffer` may overlap, so we must + * do this before any other writes into the output buffer. + * Then copy the header & padding into the output buffer. + */ + memmove(outDictContent, customDictContent, dictContentSize); + memcpy(outDictHeader, header, hSize); + memset(outDictPadding, 0, paddingSize); + return dictSize; } } diff --git a/zstd/zstd.h b/zstd/zstd.h index 222339d..b8c0644 100644 --- a/zstd/zstd.h +++ b/zstd/zstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -7,34 +7,73 @@ * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ -#if defined (__cplusplus) -extern "C" { -#endif #ifndef ZSTD_H_235446 #define ZSTD_H_235446 -/* ====== Dependency ======*/ -#include /* INT_MAX */ + +/* ====== Dependencies ======*/ #include /* size_t */ +#include "zstd_errors.h" /* list of errors */ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#include /* INT_MAX */ +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +extern "C" { +#endif /* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE # endif #endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif #endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + /******************************************************************************* Introduction @@ -71,8 +110,8 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) /*! ZSTD_versionNumber() : @@ -109,13 +148,13 @@ ZSTDLIB_API const char* ZSTD_versionString(void); #define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, @@ -123,65 +162,106 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, int compressionLevel); /*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); + +/*====== Decompression helper functions ======*/ + /*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * When invoking this method on a skippable frame, it will return 0. + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). +/*! ZSTD_getDecompressedSize() (obsolete): + * This function is now obsolete, in favor of ZSTD_getFrameContentSize(). * Both functions work the same way, but ZSTD_getDecompressedSize() blends * "empty", "unknown" and "error" results to the same return value (0), * while ZSTD_getFrameContentSize() gives them separate return values. * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); -/*! ZSTD_findFrameCompressedSize() : +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ * `src` should point to the start of a ZSTD frame or skippable frame. * `srcSize` must be >= first frame size * @return : the compressed size of the first frame starting at `src`, * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ + * or an error code if input is invalid + * Note 1: this method is called _find*() because it's not enough to read the header, + * it may have to scan through the frame's content, to reach its end. + * Note 2: this method also works with Skippable Frames. In which case, + * it returns the size of the complete skippable frame, + * which is always equal to its content size + 8 bytes for headers. */ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ /*************************************** @@ -189,25 +269,25 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres ***************************************/ /*= Compression context * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. * Note : re-using context is just a speed / resource optimization. * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ /*! ZSTD_compressCCtx() : * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. + * they will all be reset. Only @compressionLevel remains. */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, @@ -217,38 +297,38 @@ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, /*= Decompression context * When decompressing many times, * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. + * and reuse it for each successive compression operation. * This will make workload friendlier for system's memory. * Use one context per thread for parallel execution. */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ /*! ZSTD_decompressDCtx() : * Same as ZSTD_decompress(), * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. + * Compatible with sticky parameters (see below). */ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*************************************** -* Advanced compression API -***************************************/ +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ /* API design : * Parameters are pushed one by one into an existing context, * using ZSTD_CCtx_set*() functions. * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . * * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. */ @@ -266,7 +346,6 @@ typedef enum { ZSTD_fast=1, Only the order (from fast to strong) is guaranteed */ } ZSTD_strategy; - typedef enum { /* compression parameters @@ -333,6 +412,18 @@ typedef enum { * resulting in stronger and slower compression. * Special: value 0 means "use default strategy". */ + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ /* LDM mode parameters */ ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. * This parameter is designed to improve compression ratio @@ -389,7 +480,7 @@ typedef enum { ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. * The minimum size is automatically and transparently enforced. */ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. @@ -412,13 +503,18 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize * ZSTD_c_srcSizeHint * ZSTD_c_enableDedicatedDictSearch * ZSTD_c_stableInBuffer * ZSTD_c_stableOutBuffer * ZSTD_c_blockDelimiters * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -428,13 +524,21 @@ typedef enum { ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ ZSTD_c_experimentalParam7=1004, ZSTD_c_experimentalParam8=1005, ZSTD_c_experimentalParam9=1006, ZSTD_c_experimentalParam10=1007, ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 } ZSTD_cParameter; typedef struct { @@ -497,7 +601,7 @@ typedef enum { * They will be used to compress next frame. * Resetting session never fails. * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. + * This also removes any reference to any dictionary or external sequence producer. * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) * - Both : similar to resetting the session, followed by resetting parameters. @@ -506,11 +610,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); /*! ZSTD_compress2() : * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). * ZSTD_compress2() always starts a new frame. * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ @@ -519,9 +625,9 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, const void* src, size_t srcSize); -/*************************************** -* Advanced decompression API -***************************************/ +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ /* The advanced API pushes parameters one by one into an existing DCtx context. * Parameters are sticky, and remain valid for all following frames @@ -547,13 +653,17 @@ typedef enum { * ZSTD_d_stableOutBuffer * ZSTD_d_forceIgnoreChecksum * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000, ZSTD_d_experimentalParam2=1001, ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 } ZSTD_dParameter; @@ -608,14 +718,14 @@ typedef struct ZSTD_outBuffer_s { * A ZSTD_CStream object is required to track streaming operation. * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. * * For parallel execution, use one separate ZSTD_CStream per thread. * * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. * * Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. +* it will reuse the same sticky parameters as previous compression session. * When in doubt, it's recommended to fully initialize the context before usage. * Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), * ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to @@ -667,7 +777,7 @@ typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ /*===== ZSTD_CStream management functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ /*===== Streaming compression functions =====*/ typedef enum { @@ -683,7 +793,7 @@ typedef enum { : note : multithreaded compression will block to flush as much output as possible. */ } ZSTD_EndDirective; -/*! ZSTD_compressStream2() : +/*! ZSTD_compressStream2() : Requires v1.4.0+ * Behaves about the same as ZSTD_compressStream, with additional control on end directive. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) @@ -704,6 +814,11 @@ typedef enum { * only ZSTD_e_end or ZSTD_e_flush operations are allowed. * Before starting a new compression job, or changing compression parameters, * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) */ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, @@ -729,11 +844,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output /* ***************************************************************************** - * This following is a legacy streaming API. + * This following is a legacy streaming API, available since v1.0+ . * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. ******************************************************************************/ /*! @@ -742,6 +855,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); /*! @@ -762,7 +878,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. +* ZSTD_DStream objects can be re-employed multiple times. * * Use ZSTD_initDStream() to start a new decompression operation. * @return : recommended first input size @@ -772,33 +888,63 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * The function will update both `pos` fields. * If `input.pos < input.size`, some input has not been consumed. * It's up to the caller to present again remaining data. +* * The function tries to flush all data decoded immediately, respecting output buffer size. * If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. * @return : 0 when a frame is completely decoded and fully flushed, * or an error code, which can be tested using ZSTD_isError(), * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : * the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. +* that will never request more than the remaining content of the compressed frame. * *******************************************************************************/ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ /*===== ZSTD_DStream management functions =====*/ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ /*===== Streaming decompression functions =====*/ -/* This function is redundant with the advanced API and equivalent to: +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. * + * Note : This function is redundant with the advanced API and equivalent to: * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_refDDict(zds, NULL); */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ @@ -811,7 +957,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output /*! ZSTD_compress_usingDict() : * Compression at an explicit compression level using a Dictionary. * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). + * or a buffer with specified information (see zdict.h). * Note : This function loads the dictionary, resulting in significant startup delay. * It's intended for a dictionary used only once. * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ @@ -854,7 +1000,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize int compressionLevel); /*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : @@ -876,7 +1023,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict; ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); /*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); /*! ZSTD_decompress_usingDDict() : @@ -892,24 +1040,30 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, * Dictionary helper functions *******************************/ -/*! ZSTD_getDictID_fromDict() : +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ * Provides the dictID stored within dictionary. * if @return == 0, the dictionary is not conformant with Zstandard specification. * It can still be loaded, but as a content-only dictionary. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); -/*! ZSTD_getDictID_fromDDict() : +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ * Provides the dictID of the dictionary loaded into `ddict`. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); -/*! ZSTD_getDictID_fromFrame() : +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ * Provides the dictID required to decompressed the frame stored within `src`. * If @return == 0, the dictID could not be decoded. * This could for one of the following reasons : * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - This is not a Zstandard frame. @@ -918,23 +1072,26 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /******************************************************************************* - * Advanced dictionary and prefix API + * Advanced dictionary and prefix API (Requires v1.4.0+) * * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. ******************************************************************************/ -/*! ZSTD_CCtx_loadDictionary() : +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ * Create an internal CDict from `dict` buffer. * Decompression will have to use same dictionary. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. * Note 2 : Loading a dictionary involves building tables. * It's also a CPU consuming operation, with non-negligible impact on latency. * Tables are dependent on compression parameters, and for this reason, @@ -943,11 +1100,15 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. * In such a case, dictionary buffer must outlive its users. * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. * Note that compression parameters are enforced from within CDict, * and supersede any compression parameter previously set within CCtx. * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. @@ -960,12 +1121,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); -/*! ZSTD_CCtx_refPrefix() : +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ * Reference a prefix (single-usage dictionary) for next compressed frame. * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). * Decompression will need same prefix to properly regenerate data. * Compressing with a prefix is similar in outcome as performing a diff and compressing it, * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary * Note 1 : Prefix buffer is referenced. It **must** outlive compression. @@ -981,10 +1143,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * meaning "return to no-dictionary mode". @@ -998,7 +1160,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -/*! ZSTD_DCtx_refDDict() : +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ * Reference a prepared dictionary, to be used to decompress next frames. * The dictionary remains active for decompression of future frames using same DCtx. * @@ -1008,15 +1170,16 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s * The memory for the table is allocated on the first call to refDDict, and can be * freed with ZSTD_freeDCtx(). * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. * Special: referencing a NULL DDict means "return to no-dictionary mode". * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. */ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -/*! ZSTD_DCtx_refPrefix() : +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ * Reference a prefix (single-usage dictionary) to decompress next frame. * This is the reverse operation of ZSTD_CCtx_refPrefix(), * and must use the same prefix as the one used during compression. @@ -1037,7 +1200,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, /* === Memory management === */ -/*! ZSTD_sizeof_*() : +/*! ZSTD_sizeof_*() : Requires v1.4.0+ * These functions give the _current_ memory usage of selected object. * Note that object memory usage can evolve (increase or decrease) over time. */ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); @@ -1047,6 +1210,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); +#if defined (__cplusplus) +} +#endif + #endif /* ZSTD_H_235446 */ @@ -1062,6 +1229,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + /**************************************************************************************** * experimental API (static linking only) **************************************************************************************** @@ -1096,6 +1278,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_STRATEGY_MIN ZSTD_fast #define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ #define ZSTD_OVERLAPLOG_MIN 0 @@ -1119,14 +1302,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) /* Advanced parameter bounds */ -#define ZSTD_TARGETCBLOCKSIZE_MIN 64 +#define ZSTD_TARGETCBLOCKSIZE_MIN 1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */ #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX #define ZSTD_SRCSIZEHINT_MIN 0 #define ZSTD_SRCSIZEHINT_MAX INT_MAX -/* internal */ -#define ZSTD_HASHLOG3_MAX 17 - /* --- Advanced types --- */ @@ -1164,7 +1344,7 @@ typedef struct { * * Note: This field is optional. ZSTD_generateSequences() will calculate the value of * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * sequence provider perspective. For example, ZSTD_compressSequences() does not * use this 'rep' field at all (as of now). */ } ZSTD_Sequence; @@ -1268,9 +1448,19 @@ typedef enum { ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. + * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto), + * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_ParamSwitch_e; +#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */ /*************************************** -* Frame size functions +* Frame header and size functions ***************************************/ /*! ZSTD_findDecompressedSize() : @@ -1294,7 +1484,7 @@ typedef enum { * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to * read each contained frame header. This is fast as most of the data is skipped, * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_decompressBound() : * `src` should point to the start of a series of ZSTD encoded and/or skippable frames @@ -1309,41 +1499,137 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: * upper-bound = # blocks * min(128 KB, Window_Size) */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); /*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX. * @return : size of the Frame Header, * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e; +#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */ +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */ + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_FrameHeader; +#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */ + +/*! ZSTD_getFrameHeader() : + * decode Frame Header into `zfhPtr`, or requires larger `srcSize`. + * @return : 0 => header is complete, `zfhPtr` is correctly filled, + * >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize); +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; + ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */ + ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */ +} ZSTD_SequenceFormat_e; +#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */ + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); /*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsCapacity The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. * * Each block will end with a dummy sequence * with offset == 0, matchLength == 0, and litLength == length of last literals. * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) * simply acts as a block delimiter. * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). */ - -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize); /*! ZSTD_mergeBlockDelimiters() : * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. + * by merging them into the literals of the next sequence. * * As such, the final generated result has no explicit representation of block boundaries, * and the final last literals segment is not represented in the sequences. @@ -1352,11 +1638,13 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters * @return : number of sequences left after merging */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); /*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.). * The entire source is compressed into a single frame. * * The compression behavior changes based on cctx params. In particular: @@ -1365,11 +1653,17 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se * the block size derived from the cctx, and sequences may be split. This is the default setting. * * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes + * using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit + * can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation. + * By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10). + * ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction. + * + * If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) and then bail out and return an error. * * In addition to the two adjustable experimental params, there are other important cctx params. * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. @@ -1377,31 +1671,81 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. + * Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused. + * Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history. + * @return : final compressed size, or a ZSTD error code. */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_compressSequencesAndLiterals() : + * This is a variant of ZSTD_compressSequences() which, + * instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), + * aka all the literals, already extracted and laid out into a single continuous buffer. + * This can be useful if the process generating the sequences also happens to generate the buffer of literals, + * thus skipping an extraction + caching stage. + * It's a speed optimization, useful when the right conditions are met, + * but it also features the following limitations: + * - Only supports explicit delimiter mode + * - Currently does not support Sequences validation (so input Sequences are trusted) + * - Not compatible with frame checksum, which must be disabled + * - If any block is incompressible, will fail and return an error + * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. + * - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals. + * @litBufCapacity must be at least 8 bytes larger than @litSize. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t litBufCapacity, + size_t decompressedSize); /*! ZSTD_writeSkippableFrame() : * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, + * so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. * * Returns an error if destination buffer is not large enough, if the source size is not representable * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). * * @return : number of bytes written or a ZSTD error. */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer. + * + * The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. + * This can be NULL if the caller is not interested in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + /*************************************** @@ -1411,58 +1755,69 @@ ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, /*! ZSTD_estimate*() : * These functions make it possible to estimate memory usage * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. * * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. * The estimate will assume the input may be arbitrarily large, * which is the worst case. * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. * - * Note 2 : only single-threaded compression is supported. + * Note : only single-threaded compression is supported. * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); /*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); /*! ZSTD_estimate?DictSize() : * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); /*! ZSTD_initStatic*() : * Initialize an object using a pre-allocated fixed-size buffer. @@ -1485,20 +1840,20 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e * Limitation 2 : static cctx currently not compatible with multi-threading. * Limitation 3 : static dctx is incompatible with legacy support. */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( void* workspace, size_t workspaceSize, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams); -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( void* workspace, size_t workspaceSize, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, @@ -1517,46 +1872,54 @@ static #ifdef __GNUC__ __attribute__((__unused__)) #endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem); -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. */ typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); /* * This API is temporary and is expected to change or disappear in the future! */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, const ZSTD_CCtx_params* cctxParams, ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -1573,28 +1936,22 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( * As a consequence, `dictBuffer` **must** outlive CDict, * and its content must remain unmodified throughout the lifetime of CDict. * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); - -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_checkCParams() : * Ensure param values remain within authorized range. * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. @@ -1602,23 +1959,48 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); * `dictSize` must be `0` when there is no dictionary. * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); /*! ZSTD_compress_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. + * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, @@ -1628,18 +2010,18 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, /*! ZSTD_CCtx_loadDictionary_byReference() : * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); /*! ZSTD_CCtx_loadDictionary_advanced() : * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over * how to load the dictionary (by copy ? by reference ?) * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); /*! ZSTD_CCtx_refPrefix_advanced() : * Same as ZSTD_CCtx_refPrefix(), but gives finer control over * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); /* === experimental parameters === */ /* these parameters can be used with ZSTD_setParameter() @@ -1678,17 +2060,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * See the comments on that enum for an explanation of the feature. */ #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. +/* Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - /* User's best guess of source size. * Hint is not valid when srcSizeHint == 0. * There is no guarantee that hint is close to actual source size, @@ -1743,7 +2126,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * * Note that this means that the CDict tables can no longer be copied into the * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. + * usable. The dictionary can only be attached or reloaded. * * In general, you should expect compression to be faster--sometimes very much * so--and CDict creation to be slightly slower. Eventually, we will probably @@ -1755,13 +2138,16 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * Experimental parameter. * Default is 0 == disabled. Set to 1 to enable. * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. * * When this flag is enabled zstd won't allocate an input window buffer, * because the user guarantees it can reference the ZSTD_inBuffer until @@ -1769,18 +2155,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * avoid the memcpy() from the input buffer to the input window buffer. * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. + * memory. However, compression WILL fail if conditions are not respected. * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. + * but passing this flag tells zstd to rely on user provided buffer instead. */ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 @@ -1818,26 +2201,167 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre /* ZSTD_c_validateSequences * Default is 0 == disabled. Set to 1 to enable sequence validation. * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * For use with sequence compression API: ZSTD_compressSequences*(). + * Designates whether or not provided sequences are validated within ZSTD_compressSequences*() * during function execution. * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. + * When Sequence validation is disabled (default), Sequences are compressed as-is, + * so they must correct, otherwise it would result in a corruption error. * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * Sequence validation adds some protection, by ensuring that all values respect boundary conditions. + * If a Sequence is detected invalid (see doc/zstd_compression_format.md for * specifics regarding offset/matchlength requirements) then the function will bail out and * return an error. - * */ #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 +/* ZSTD_c_blockSplitterLevel + * note: this parameter only influences the first splitter stage, + * which is active before producing the sequences. + * ZSTD_c_splitAfterSequences controls the next splitter stage, + * which is active after sequence production. + * Note that both can be combined. + * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included. + * 0 means "auto", which will select a value depending on current ZSTD_c_strategy. + * 1 means no splitting. + * Then, values from 2 to 6 are sorted in increasing cpu load order. + * + * Note that currently the first block is never split, + * to ensure expansion guarantees in presence of incompressible data. + */ +#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6 +#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20 + +/* ZSTD_c_splitAfterSequences + * This is a stronger splitter algorithm, + * based on actual sequences previously produced by the selected parser. + * It's also slower, and as a consequence, mostly used for high compression levels. + * While the post-splitter does overlap with the pre-splitter, + * both can nonetheless be combined, + * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX, + * resulting in higher compression ratio than just one of them. + * + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_repcodeResolution + * This parameter only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters (may change in the future). + * + * This parameter affects how zstd parses external sequences, + * provided via the ZSTD_compressSequences*() API + * or from an external block-level sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets within + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences*() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level (currently: level<10 disables, level>=10 enables). + */ +#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19 +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */ + + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. * @return : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); /*! ZSTD_CCtx_params : @@ -1852,32 +2376,32 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter * These parameters will be applied to * all subsequent frames. * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. * * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() * for static allocation of CCtx for single-threaded compression. */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ /*! ZSTD_CCtxParams_reset() : * Reset params to default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); /*! ZSTD_CCtxParams_init() : * Initializes the compression parameters of cctxParams according to * compression level. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); /*! ZSTD_CCtxParams_init_advanced() : * Initializes the compression and frame parameters of cctxParams according to * params. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); -/*! ZSTD_CCtxParams_setParameter() : +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ * Similar to ZSTD_CCtx_setParameter. * Set one compression parameter, selected by enum ZSTD_cParameter. * Parameters must be applied to a ZSTD_CCtx using @@ -1885,14 +2409,14 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, Z * @result : a code representing success or failure (which can be tested with * ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); /*! ZSTD_CCtxParams_getParameter() : * Similar to ZSTD_CCtx_getParameter. * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. * @result : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); /*! ZSTD_CCtx_setParametersUsingCCtxParams() : * Apply a set of ZSTD_CCtx_params to the compression context. @@ -1901,7 +2425,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, * if nbWorkers>=1, new parameters will be picked up at next job, * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); /*! ZSTD_compressStream2_simpleArgs() : @@ -1910,7 +2434,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( * This variant might be helpful for binders from dynamic languages * which have troubles handling structures containing memory pointers. */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos, @@ -1926,33 +2450,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); /*! ZSTD_createDDict_byReference() : * Create a digested dictionary, ready to start decompression operation without startup delay. * Dictionary content is referenced, and therefore stays in dictBuffer. * It is important that dictBuffer outlives DDict, * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); /*! ZSTD_DCtx_loadDictionary_byReference() : * Same as ZSTD_DCtx_loadDictionary(), * but references `dict` content instead of copying it into `dctx`. * This saves memory if `dict` remains around., * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /*! ZSTD_DCtx_loadDictionary_advanced() : * Same as ZSTD_DCtx_loadDictionary(), * but gives direct control over * how to load the dictionary (by copy ? by reference ?) * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); /*! ZSTD_DCtx_refPrefix_advanced() : * Same as ZSTD_DCtx_refPrefix(), but gives finer control over * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); /*! ZSTD_DCtx_setMaxWindowSize() : * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. @@ -1961,14 +2485,14 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); /*! ZSTD_DCtx_getParameter() : * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, * and store it into int* value. * @return : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); /* ZSTD_d_format * experimental parameter, @@ -1988,7 +2512,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param * in the range [dst, dst + pos) MUST not be modified during decompression * or you will get data corruption. * - * When this flags is enabled zstd won't allocate an output buffer, because + * When this flag is enabled zstd won't allocate an output buffer, because * it can write directly to the ZSTD_outBuffer, but it will still allocate * an input buffer large enough to fit any compressed block. This will also * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. @@ -2041,13 +2565,43 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param */ #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + /*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, * such ZSTD_f_zstd1_magicless for example. * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /*! ZSTD_decompressStream_simpleArgs() : * Same as ZSTD_decompressStream(), @@ -2055,7 +2609,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); * This can be helpful for binders from dynamic languages * which have troubles handling structures containing memory pointers. */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos); @@ -2071,7 +2625,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( /*===== Advanced Streaming compression functions =====*/ /*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); @@ -2080,15 +2634,16 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); @@ -2097,81 +2652,85 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, * dict == NULL or dictSize < 8, in which case no dict is used. * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } + * ZSTD_CCtx_setParams(zcs, params); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. * pledgedSrcSize must be correct. * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); * * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } + * ZSTD_CCtx_setFParams(zcs, fParams); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. * * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. * Note that zcs must be init at least once before using ZSTD_resetCStream(). * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); typedef struct { @@ -2189,7 +2748,7 @@ typedef struct { * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. * Aggregates progression inside active worker threads. */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); /*! ZSTD_toFlushNow() : * Tell how many bytes are ready to be flushed immediately. @@ -2204,7 +2763,7 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx * therefore flush speed is limited by production speed of oldest job * irrespective of the speed of concurrent (and newer) jobs. */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); /*===== Advanced Streaming decompression functions =====*/ @@ -2216,9 +2775,9 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /*! * This function is deprecated, and is equivalent to: @@ -2227,27 +2786,211 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic * ZSTD_DCtx_refDDict(zds, ddict); * * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /*! * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * reuse decompression parameters from previous init; saves dictionary loading */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); /********************************************************************* -* Buffer-less and synchronous inner streaming functions +* Buffer-less and synchronous inner streaming functions (DEPRECATED) * -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. ********************************************************************* */ /** @@ -2255,12 +2998,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. + ZSTD_CCtx object can be reused multiple times within successive compression operations. Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : @@ -2278,37 +3019,49 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. */ /*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ /** Buffer-less streaming decompression (synchronous mode) A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. + A ZSTD_DCtx object can be reused multiple times. First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. Data fragment must be large enough to ensure successful decoding. `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. errorCode, which can be tested using ZSTD_isError(). - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame, such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. As a consequence, check that values remain within valid application range. @@ -2324,7 +3077,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci The most memory efficient way is to use a round buffer of sufficient size. Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). + which can return an error code if required value is too large for current system (in 32-bits mode). In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, up to the moment there is not enough room left in the buffer to guarantee decoding another full block, which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. @@ -2344,7 +3097,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. It can also be an error code, which can be tested with ZSTD_isError(). @@ -2367,49 +3120,42 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci */ /*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); -/* ============================ */ -/** Block level API */ -/* ============================ */ +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ /*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. @@ -2420,7 +3166,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - It is necessary to init context before starting + compression : any ZSTD_compressBegin*() variant, including with dictionary + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks + For inputs larger than a single block, consider using regular ZSTD_compress() instead. @@ -2437,14 +3182,17 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); */ /*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - - -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ #if defined (__cplusplus) } #endif + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ diff --git a/zstd/zstddeclib.c b/zstd/zstddeclib.c index de206f6..875d3b7 100644 --- a/zstd/zstddeclib.c +++ b/zstd/zstddeclib.c @@ -4,11 +4,11 @@ * * Generate using: * \code - * combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c + * python combine.py -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c * \endcode */ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -25,8 +25,12 @@ * Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also * defined in mem.h (breaking C99 compatibility). * - * Note: the undefs for xxHash allow Zstd's implementation to coinside with with + * Note: the undefs for xxHash allow Zstd's implementation to coincide with * standalone xxHash usage (with global defines). + * + * Note: if you enable ZSTD_LEGACY_SUPPORT the combine.py script will need + * re-running without the "-x legacy/zstd_legacy.h" option (it excludes the + * legacy support at the source level). */ #define DEBUGLEVEL 0 #define MEM_MODULE @@ -39,12 +43,14 @@ #define ZSTD_LEGACY_SUPPORT 0 #define ZSTD_STRIP_ERROR_STRINGS #define ZSTD_TRACE 0 +/* TODO: Can't amalgamate ASM function */ +#define ZSTD_DISABLE_ASM 1 /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC /**** start inlining common/zstd_deps.h ****/ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -69,6 +75,18 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON +/* Even though we use qsort_r only for the dictionary builder, the macro + * _GNU_SOURCE has to be declared *before* the inclusion of any standard + * header and the script 'combine.sh' combines the whole zstd source code + * in a single file. + */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include #include #include @@ -160,7 +178,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -182,7 +200,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -213,10 +231,6 @@ #ifndef DEBUG_H_12987983217 #define DEBUG_H_12987983217 -#if defined (__cplusplus) -extern "C" { -#endif - /* static assert is triggered at compile time, leaving no runtime artefact. * static assert only works with compile-time constants. @@ -266,34 +280,44 @@ extern int g_debuglevel; /* the variable is only declared, It's useful when enabling very verbose levels on selective conditions (such as position in src) */ -# define RAWLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__VA_ARGS__); \ - } } -# define DEBUGLOG(l, ...) { \ - if (l<=g_debuglevel) { \ - ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ - ZSTD_DEBUG_PRINT(" \n"); \ - } } +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) + +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) #else -# define RAWLOG(l, ...) {} /* disabled */ -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -#if defined (__cplusplus) -} +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ #endif #endif /* DEBUG_H_12987983217 */ /**** ended inlining debug.h ****/ +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ int g_debuglevel = DEBUGLEVEL; +#endif /**** ended inlining common/debug.c ****/ /**** start inlining common/entropy_common.c ****/ /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -310,7 +334,7 @@ int g_debuglevel = DEBUGLEVEL; ***************************************/ /**** start inlining mem.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -322,17 +346,13 @@ int g_debuglevel = DEBUGLEVEL; #ifndef MEM_H_MODULE #define MEM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - /*-**************************************** * Dependencies ******************************************/ #include /* size_t, ptrdiff_t */ /**** start inlining compiler.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -344,6 +364,182 @@ extern "C" { #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include + +/**** start inlining portability_macros.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Compile time determination of BMI2 support */ +#ifndef STATIC_BMI2 +# if defined(__BMI2__) +# define STATIC_BMI2 1 +# elif defined(_MSC_VER) && defined(__AVX2__) +# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */ +# endif +#endif + +#ifndef STATIC_BMI2 +# define STATIC_BMI2 0 +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 +# if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \ + && !defined(__BMI2__) +# define DYNAMIC_BMI2 1 +# else +# define DYNAMIC_BMI2 0 +# endif +#endif + +/** + * Only enable assembly for GNU C compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS / Win32, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ +/**** ended inlining portability_macros.h ****/ + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -356,7 +552,7 @@ extern "C" { # define INLINE_KEYWORD #endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -373,7 +569,7 @@ extern "C" { /** On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). - This explictly marks such functions as __cdecl so that the code will still compile + This explicitly marks such functions as __cdecl so that the code will still compile if a CC other than __cdecl has been made the default. */ #if defined(_MSC_VER) @@ -382,12 +578,19 @@ extern "C" { # define WIN_CDECL #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR /** * HINT_INLINE is used to help the compiler generate better code. It is *not* * used for "templates", so it can be tweaked based on the compilers @@ -402,21 +605,37 @@ extern "C" { #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 # define HINT_INLINE static INLINE_KEYWORD #else -# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +# define HINT_INLINE FORCE_INLINE_TEMPLATE #endif -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ #if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined(__IAR_SYSTEMS_ICC__) +# define MEM_STATIC static inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline #else -# define UNUSED_ATTR +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif #endif /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# if defined(__GNUC__) || defined(__ICCARM__) +# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -425,37 +644,25 @@ extern "C" { /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) #endif -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X86)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ #else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) @@ -463,28 +670,30 @@ extern "C" { # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) # else -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ # endif #endif /* NO_PREFETCH */ #define CACHELINE_SIZE 64 -#define PREFETCH_AREA(p, s) { \ - const char* const _ptr = (const char*)(p); \ - size_t const _size = (size_t)(s); \ - size_t _pos; \ - for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH_L2(_ptr + _pos); \ - } \ -} +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) /* vectorization - * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ -#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) # else @@ -507,6 +716,12 @@ extern "C" { #define UNLIKELY(x) (x) #endif +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ @@ -517,39 +732,198 @@ extern "C" { # pragma warning(disable : 4324) /* disable: C4324: padded structure */ #endif -/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ -#ifndef STATIC_BMI2 -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) -# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 -# define STATIC_BMI2 1 +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__AVX2__) +# define ZSTD_ARCH_X86_AVX2 +# endif +# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_AVX2) +# include +# endif +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment +*****************************************************************/ + +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +#ifndef ZSTD_ALIGNED +/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */ +# if defined(__GNUC__) || defined(__clang__) +# define ZSTD_ALIGNED(a) __attribute__((aligned(a))) +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define ZSTD_ALIGNED(a) _Alignas(a) +#elif defined(_MSC_VER) +# define ZSTD_ALIGNED(n) __declspec(align(n)) +# else + /* this compiler will require its own alignment instruction */ +# define ZSTD_ALIGNED(...) +# endif +#endif /* ZSTD_ALIGNED */ + + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) # endif -# endif -#endif - -#ifndef STATIC_BMI2 - #define STATIC_BMI2 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 -#endif - -/* detects whether we are being compiled under msan */ -#ifndef ZSTD_MEMORY_SANITIZER -# if __has_feature(memory_sanitizer) -# define ZSTD_MEMORY_SANITIZER 1 # else -# define ZSTD_MEMORY_SANITIZER 0 +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR # endif #endif -#if ZSTD_MEMORY_SANITIZER +/** + * Helper function to perform a wrapped pointer difference without triggering + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) /* Not all platforms that support msan provide sanitizers/msan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -568,20 +942,13 @@ void __msan_poison(const volatile void *a, size_t size); /* Returns the offset of the first (at least partially) poisoned byte in the memory range, or -1 if the whole range is good. */ intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); #endif -/* detects whether we are being compiled under asan */ -#ifndef ZSTD_ADDRESS_SANITIZER -# if __has_feature(address_sanitizer) -# define ZSTD_ADDRESS_SANITIZER 1 -# elif defined(__SANITIZE_ADDRESS__) -# define ZSTD_ADDRESS_SANITIZER 1 -# else -# define ZSTD_ADDRESS_SANITIZER 0 -# endif -#endif - -#if ZSTD_ADDRESS_SANITIZER +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -631,15 +998,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #if defined(_MSC_VER) /* Visual Studio */ # include /* _byteswap_ulong */ # include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#elif defined(__ICCARM__) +# include #endif /*-************************************************************** @@ -652,6 +1012,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # include /* intptr_t */ # endif typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; typedef uint16_t U16; typedef int16_t S16; typedef uint32_t U32; @@ -664,6 +1026,8 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); # error "this implementation requires char to be exactly 8-bit type" #endif typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; #if USHRT_MAX != 65535 # error "this implementation requires short to be exactly 16-bit type" #endif @@ -680,7 +1044,6 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); typedef signed long long S64; #endif - /*-************************************************************** * Memory I/O API *****************************************************************/ @@ -730,23 +1093,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -756,8 +1111,24 @@ MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_X64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__ + return 1; +#else const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; +#endif } #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) @@ -775,30 +1146,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -842,6 +1202,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -849,14 +1217,25 @@ MEM_STATIC U32 MEM_swap32(U32 in) #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); +#elif defined(__ICCARM__) + return __REV(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + MEM_STATIC U64 MEM_swap64(U64 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -865,14 +1244,7 @@ MEM_STATIC U64 MEM_swap64(U64 in) || (defined(__clang__) && __has_builtin(__builtin_bswap64)) return __builtin_bswap64(in); #else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); + return MEM_swap64_fallback(in); #endif } @@ -909,7 +1281,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) MEM_STATIC U32 MEM_readLE24(const void* memPtr) { - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); } MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) @@ -1019,16 +1391,11 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) /* code only tested on 32 and 64 bits systems */ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - -#if defined (__cplusplus) -} -#endif - #endif /* MEM_H_MODULE */ /**** ended inlining mem.h ****/ /**** start inlining error_private.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1042,18 +1409,12 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si #ifndef ERROR_H_MODULE #define ERROR_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif - - /* **************************************** * Dependencies ******************************************/ -/**** skipping file: zstd_deps.h ****/ -/**** start inlining zstd_errors.h ****/ +/**** start inlining ../zstd_errors.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -1069,24 +1430,32 @@ extern "C" { extern "C" { #endif -/*===== dependency =====*/ -#include /* size_t */ - - /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDERRORLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE # endif #endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE #endif /*-********************************************* @@ -1112,14 +1481,18 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_cannotProduce_uncompressedBlock = 49, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -1127,18 +1500,18 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare with enum list published above */ -ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ @@ -1147,8 +1520,10 @@ ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Sa #endif #endif /* ZSTD_ERRORS_H_398273423 */ -/**** ended inlining zstd_errors.h ****/ - +/**** ended inlining ../zstd_errors.h ****/ +/**** skipping file: compiler.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: zstd_deps.h ****/ /* **************************************** * Compiler-specific @@ -1183,8 +1558,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /* check and forward error code */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) /*-**************************************** @@ -1198,9 +1578,86 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) return ERR_getErrorString(ERR_getErrorCode(code)); } -#if defined (__cplusplus) +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; } -#endif + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) #endif /* ERROR_H_MODULE */ /**** ended inlining error_private.h ****/ @@ -1209,7 +1666,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1219,11 +1676,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ - -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef FSE_H #define FSE_H @@ -1233,7 +1685,6 @@ extern "C" { ******************************************/ /**** skipping file: zstd_deps.h ****/ - /*-***************************************** * FSE_PUBLIC_API : control library symbols visibility ******************************************/ @@ -1261,34 +1712,6 @@ extern "C" { FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ @@ -1299,20 +1722,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -1372,8 +1781,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -1449,23 +1856,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -1497,15 +1888,14 @@ If there is an error, the function will return an error code, which can be teste #endif /* FSE_H */ + #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) #define FSE_H_FSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ /**** start inlining bitstream.h ****/ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -1518,9 +1908,6 @@ If there is an error, the function will return an error code, which can be teste #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE -#if defined (__cplusplus) -extern "C" { -#endif /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -1534,14 +1921,220 @@ extern "C" { /**** skipping file: compiler.h ****/ /**** skipping file: debug.h ****/ /**** skipping file: error_private.h ****/ +/**** start inlining bits.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +/**** skipping file: mem.h ****/ + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctz(val); +#else + return ZSTD_countTrailingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +#if defined(_MSC_VER) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_clz(val); +#else + return ZSTD_countLeadingZeros32_fallback(val); +#endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +#elif defined(__ICCARM__) + return (unsigned)__builtin_ctzll(val); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +#if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + __assume(0); /* Should not reach this code path */ + } +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +#elif defined(__ICCARM__) + return (unsigned)(__builtin_clzll(val)); +#else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +#endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ +/**** ended inlining bits.h ****/ /*========================================= * Target specific =========================================*/ #ifndef ZSTD_NO_INTRINSICS -# if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ # elif defined(__ICCARM__) # include # endif @@ -1555,12 +2148,13 @@ extern "C" { /*-****************************************** * bitStream encoding API (write forward) ********************************************/ +typedef size_t BitContainerType; /* bitStream can mix input from multiple sources. * A critical property of these streams is that they encode and decode in **reverse** direction. * So the first bit sequence you add will be the last to be read, like a LIFO stack. */ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitPos; char* startPtr; char* ptr; @@ -1568,7 +2162,7 @@ typedef struct { } BIT_CStream_t; MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); @@ -1577,7 +2171,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems. * Writing data into memory is an explicit operation, performed by the flushBits function. * Hence keep track how many bits are potentially stored into local register to avoid register overflow. * After a flushBits, a maximum of 7 bits might still be stored into local register. @@ -1594,28 +2188,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * bitStream decoding API (read backward) **********************************************/ typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitsConsumed; const char* ptr; const char* start; const char* limitPtr; } BIT_DStream_t; -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /* Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. @@ -1627,7 +2221,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /*-**************************************** * unsafe API ******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits); /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); @@ -1636,42 +2230,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - unsigned long r = 0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -1701,16 +2259,31 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits) +{ +#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS) +# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__) + return _bzhi_u64(bitContainer, nbBits); +# else + DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32)); + return _bzhi_u32(bitContainer, nbBits); +# endif +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -1718,7 +2291,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, * works only if `value` is _clean_, * meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, - size_t value, unsigned nbBits) + BitContainerType value, unsigned nbBits) { assert((value>>nbBits) == 0); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); @@ -1765,7 +2338,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_flushBits(bitC); if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); + return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); } @@ -1789,35 +2362,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - /* fall-through */ + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - /* fall-through */ + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - /* fall-through */ + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - /* fall-through */ + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - /* fall-through */ + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - /* fall-through */ + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -1826,26 +2399,25 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ assert(nbBits < BIT_MASK_SIZE); - return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; -} - -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X64) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); #else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; #endif } @@ -1855,7 +2427,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 co * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -1871,14 +2443,14 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U3 /*! BIT_lookBitsFast() : * unsafe version; only works if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) { U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; assert(nbBits >= 1); return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -1887,23 +2459,38 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBits(bitD, nbBits); + BitContainerType const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); return value; } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { - size_t const value = BIT_lookBitsFast(bitD, nbBits); + BitContainerType const value = BIT_lookBitsFast(bitD, nbBits); assert(nbBits >= 1); BIT_skipBits(bitD, nbBits); return value; } +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStreamFast() : * Similar to BIT_reloadDStream(), but with two differences: * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! @@ -1914,31 +2501,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) { if (UNLIKELY(bitD->ptr < bitD->limitPtr)) return BIT_DStream_overflow; - assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStream_internal(bitD); } /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . - * This function is safe, it guarantees it will not read beyond src buffer. + * This function is safe, it guarantees it will not never beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); if (bitD->ptr >= bitD->limitPtr) { - return BIT_reloadDStreamFast(bitD); + return BIT_reloadDStream_internal(bitD); } if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } - /* start < ptr < limitPtr */ + /* start < ptr < limitPtr => cautious update */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -1960,14 +2551,9 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); } -#if defined (__cplusplus) -} -#endif - #endif /* BITSTREAM_H_MODULE */ /**** ended inlining bitstream.h ****/ - /* ***************************************** * Static allocation *******************************************/ @@ -1992,24 +2578,15 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /* FSE_buildCTable_wksp() : * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. */ -#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2))) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); @@ -2018,19 +2595,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -2213,20 +2782,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); + BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) { - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog); BIT_flushBits(bitC); } /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) @@ -2379,20 +2948,13 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) - #endif /* FSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif /**** ended inlining fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ /**** start inlining huf.h ****/ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -2403,115 +2965,33 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ -#if defined (__cplusplus) -extern "C" { -#endif - #ifndef HUF_H_298734234 #define HUF_H_298734234 /* *** Dependencies *** */ /**** skipping file: zstd_deps.h ****/ - - -/* *** library symbols visibility *** */ -/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, - * HUF symbols remain "private" (internal symbols for library only). - * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ -#else -# define HUF_PUBLIC_API -#endif - - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) -#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); - -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ /**** skipping file: mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) /* *** Constants *** */ -#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ #define HUF_SYMBOLVALUE_MAX 255 -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) # error "HUF_TABLELOG_MAX is too large !" #endif @@ -2527,15 +3007,11 @@ HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, /* static allocation of HUF's Compression Table */ /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt */ -typedef struct HUF_CElt_s HUF_CElt; /* consider it an incomplete type */ -#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ -#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - HUF_CElt name[HUF_CTABLE_SIZE_U32(maxSymbolValue)] /* no final ; */ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ /* static allocation of HUF's DTable */ typedef U32 HUF_DTable; @@ -2549,25 +3025,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -2580,10 +3080,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); @@ -2592,22 +3094,24 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, @@ -2633,17 +3137,29 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); -/** HUF_getNbBits() : +/** HUF_getNbBitsFromCTable() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX - * Note 1 : is not inlined, as HUF_CElt definition is private - * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ -U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); /* * HUF_decompress() does the following: @@ -2669,88 +3185,51 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); * a required workspace size greater than that specified in the following * macro. */ -#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. - * If preferRepeat then the old table will always be used if valid. */ + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif -#endif /* HUF_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} -#endif +#endif /* HUF_H_298734234 */ /**** ended inlining huf.h ****/ +/**** skipping file: bits.h ****/ /*=== Version ===*/ @@ -2768,28 +3247,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanForward(&r, val) ? (unsigned)r : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -2837,7 +3294,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -2848,7 +3305,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -2913,7 +3370,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -2947,7 +3404,7 @@ static size_t FSE_readNCount_body_default( } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -2988,7 +3445,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -3029,21 +3486,21 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); weightTotal = 0; { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected); rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -3067,7 +3524,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) @@ -3080,20 +3537,20 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } /**** ended inlining common/entropy_common.c ****/ /**** start inlining common/error_private.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3121,9 +3578,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -3132,17 +3591,23 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } @@ -3152,7 +3617,7 @@ const char* ERR_getErrorString(ERR_enum code) /**** start inlining common/fse_decompress.c ****/ /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -3174,8 +3639,8 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_STATIC_LINKING_ONLY /**** skipping file: fse.h ****/ /**** skipping file: error_private.h ****/ -#define ZSTD_DEPS_NEED_MALLOC /**** skipping file: zstd_deps.h ****/ +/**** skipping file: bits.h ****/ /* ************************************************************** @@ -3207,19 +3672,6 @@ const char* ERR_getErrorString(ERR_enum code) #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -3248,7 +3700,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + symbolNext[s] = (U16)normalizedCounter[s]; } } } ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -3263,8 +3715,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo * all symbols have counts <= 8. We ensure we have 8 bytes at the end of * our buffer to handle the over-write. */ - { - U64 const add = 0x0101010101010101ull; + { U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0; U32 s; @@ -3275,14 +3726,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; - } - } + pos += (size_t)n; + } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -3318,7 +3768,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s= ostart); + return (size_t)(op-ostart); } +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); -} FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( void* dst, size_t dstCapacity, @@ -3470,24 +3865,34 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - FSE_DTable* const dtable = (FSE_DTable*)workSpace; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); - if (FSE_isError(NCountLength)) return NCountLength; - if (tableLog > maxLog) return ERROR(tableLog_tooLarge); - assert(NCountLength <= cSrcSize); - ip += NCountLength; - cSrcSize -= NCountLength; + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog); - wkspSize -= FSE_DTABLE_SIZE(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); - CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) ); + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); { const void* ptr = dtable; @@ -3507,7 +3912,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } @@ -3524,29 +3929,11 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; - return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); -} - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - /* Static analyzer seems unable to understand this table will be properly initialized later */ - U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); -} -#endif - - #endif /* FSE_COMMONDEFS_ONLY */ /**** ended inlining common/fse_decompress.c ****/ /**** start inlining common/zstd_common.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3561,11 +3948,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS * Dependencies ***************************************/ #define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ /**** skipping file: error_private.h ****/ /**** start inlining zstd_internal.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -3585,5467 +3971,10 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS /*-************************************* * Dependencies ***************************************/ -#if !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) -#include -#endif /**** skipping file: compiler.h ****/ -/**** skipping file: mem.h ****/ -/**** skipping file: debug.h ****/ -/**** skipping file: error_private.h ****/ -#define ZSTD_STATIC_LINKING_ONLY -/**** start inlining ../zstd.h ****/ +/**** start inlining cpu.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef ZSTD_H_235446 -#define ZSTD_H_235446 - -/* ====== Dependency ======*/ -#include /* INT_MAX */ -#include /* size_t */ - - -/* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define ZSTDLIB_VISIBILITY -# endif -#endif -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define ZSTDLIB_API ZSTDLIB_VISIBILITY -#endif - - -/******************************************************************************* - Introduction - - zstd, short for Zstandard, is a fast lossless compression algorithm, targeting - real-time compression scenarios at zlib-level and better compression ratios. - The zstd compression library provides in-memory compression and decompression - functions. - - The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), - which is currently 22. Levels >= 20, labeled `--ultra`, should be used with - caution, as they require more memory. The library also offers negative - compression levels, which extend the range of speed vs. ratio preferences. - The lower the level, the faster the speed (at the cost of compression). - - Compression can be done in: - - a single step (described as Simple API) - - a single step, reusing a context (described as Explicit context) - - unbounded multiple steps (described as Streaming compression) - - The compression ratio achievable on small data can be highly improved using - a dictionary. Dictionary compression can be performed in: - - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Bulk-processing - dictionary API) - - Advanced experimental functions can be accessed using - `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. - - Advanced experimental APIs should never be used with a dynamically-linked - library. They are not "stable"; their definitions or signatures may change in - the future. Only static linking is allowed. -*******************************************************************************/ - -/*------ Version ------*/ -#define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 9 -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) - -/*! ZSTD_versionNumber() : - * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ -ZSTDLIB_API unsigned ZSTD_versionNumber(void); - -#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE -#define ZSTD_QUOTE(str) #str -#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) -#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) - -/*! ZSTD_versionString() : - * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ -ZSTDLIB_API const char* ZSTD_versionString(void); - -/* ************************************* - * Default constant - ***************************************/ -#ifndef ZSTD_CLEVEL_DEFAULT -# define ZSTD_CLEVEL_DEFAULT 3 -#endif - -/* ************************************* - * Constants - ***************************************/ - -/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ -#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ -#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 - -#define ZSTD_BLOCKSIZELOG_MAX 17 -#define ZSTD_BLOCKSIZE_MAX (1<= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*! ZSTD_decompress() : - * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - * `dstCapacity` is an upper bound of originalSize to regenerate. - * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/*! ZSTD_getFrameContentSize() : requires v1.3.0+ - * `src` should point to the start of a ZSTD encoded frame. - * `srcSize` must be at least as large as the frame header. - * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - * @return : - decompressed size of `src` frame content, if known - * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) - * note 1 : a 0 return value means the frame is valid but "empty". - * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can rely on some implicit limit, - * as ZSTD_decompress() only needs an upper bound of decompressed size. - * (For example, data could be necessarily cut into blocks <= 16 KB). - * note 3 : decompressed size is always present when compression is completed using single-pass functions, - * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). - * note 4 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure return value fits within application's authorized limits. - * Each application can set its own limits. - * note 6 : This function replaces ZSTD_getDecompressedSize() */ -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) -ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); - -/*! ZSTD_getDecompressedSize() : - * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). - * Both functions work the same way, but ZSTD_getDecompressedSize() blends - * "empty", "unknown" and "error" results to the same return value (0), - * while ZSTD_getFrameContentSize() gives them separate return values. - * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_findFrameCompressedSize() : - * `src` should point to the start of a ZSTD frame or skippable frame. - * `srcSize` must be >= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); - - -/*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ -ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ -ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ - - -/*************************************** -* Explicit context -***************************************/ -/*= Compression context - * When compressing many times, - * it is recommended to allocate a context just once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Note : re-using context is just a speed / resource optimization. - * It doesn't change the compression ratio, which remains identical. - * Note 2 : In multi-threaded environments, - * use one different context per thread for parallel execution. - */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); -ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); - -/*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. - * Important : in order to behave similarly to `ZSTD_compress()`, - * this function compresses at requested compression level, - * __ignoring any other parameter__ . - * If any advanced parameter was set using the advanced API, - * they will all be reset. Only `compressionLevel` remains. - */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); - -/*= Decompression context - * When decompressing many times, - * it is recommended to allocate a context only once, - * and re-use it for each successive compression operation. - * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution. */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); -ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); - -/*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), - * requires an allocated ZSTD_DCtx. - * Compatible with sticky parameters. - */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/*************************************** -* Advanced compression API -***************************************/ - -/* API design : - * Parameters are pushed one by one into an existing context, - * using ZSTD_CCtx_set*() functions. - * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. - * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . - * - * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). - * - * This API supercedes all other "advanced" API entry points in the experimental section. - * In the future, we expect to remove from experimental API entry points which are redundant with this API. - */ - - -/* Compression strategies, listed from fastest to strongest */ -typedef enum { ZSTD_fast=1, - ZSTD_dfast=2, - ZSTD_greedy=3, - ZSTD_lazy=4, - ZSTD_lazy2=5, - ZSTD_btlazy2=6, - ZSTD_btopt=7, - ZSTD_btultra=8, - ZSTD_btultra2=9 - /* note : new strategies _might_ be added in the future. - Only the order (from fast to strong) is guaranteed */ -} ZSTD_strategy; - - -typedef enum { - - /* compression parameters - * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. - * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. - * Note that exact compression parameters are dynamically determined, - * depending on both compression level and srcSize (when known). - * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. - * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level does not automatically set all other compression parameters - * to default. Setting this will however eventually dynamically impact the compression - * parameters which have not been manually set. The manually set - * ones will 'stick'. */ - /* Advanced compression parameters : - * It's possible to pin down compression parameters to some specific values. - * In which case, these values are no longer dynamically selected by the compressor */ - ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. - * This will set a memory budget for streaming decompression, - * with larger values requiring more memory - * and typically compressing more. - * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "use default windowLog". - * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such size at streaming decompression stage. */ - ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. - * Resulting memory usage is (1 << (hashLog+2)). - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. - * Larger tables improve compression ratio of strategies <= dFast, - * and improve speed of strategies > dFast. - * Special: value 0 means "use default hashLog". */ - ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. - * Resulting memory usage is (1 << (chainLog+2)). - * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. - * Larger tables result in better and slower compression. - * This parameter is useless for "fast" strategy. - * It's still useful when using "dfast" strategy, - * in which case it defines a secondary probe table. - * Special: value 0 means "use default chainLog". */ - ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. - * More attempts result in better and slower compression. - * This parameter is useless for "fast" and "dFast" strategies. - * Special: value 0 means "use default searchLog". */ - ZSTD_c_minMatch=105, /* Minimum size of searched matches. - * Note that Zstandard can still find matches of smaller size, - * it just tweaks its search algorithm to look for this size and larger. - * Larger values increase compression and decompression speed, but decrease ratio. - * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. - * Note that currently, for all strategies < btopt, effective minimum is 4. - * , for all strategies > fast, effective maximum is 6. - * Special: value 0 means "use default minMatchLength". */ - ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. - * For strategies btopt, btultra & btultra2: - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger, and slower. - * For strategy fast: - * Distance between match sampling. - * Larger values make compression faster, and weaker. - * Special: value 0 means "use default targetLength". */ - ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. - * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. - * Special: value 0 means "use default strategy". */ - - /* LDM mode parameters */ - ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. - * This parameter is designed to improve compression ratio - * for large inputs, by finding large matches at long distance. - * It increases memory usage and window size. - * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB - * except when expressly set to a different value. - * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and - * compression strategy >= ZSTD_btopt (== compression level 16+) */ - ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, - * but decrease compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * default: windowlog - 7. - * Special: value 0 means "automatically determine hashlog". */ - ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. - * Special: value 0 means "use default value" (default: 64). */ - ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. - * Special: value 0 means "use default value" (default: 3). */ - ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. - * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). - * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. - * Larger values improve compression speed. - * Deviating far from default value will likely result in a compression ratio decrease. - * Special: value 0 means "automatically determine hashRateLog". */ - - /* frame parameters */ - ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression. - * This is automatically the case when using ZSTD_compress2(), - * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ - ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ - ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ - - /* multi-threading parameters */ - /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). - * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. - * In a situation where it's unknown if the linked library supports multi-threading or not, - * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. - */ - ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : - * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, - * while compression is performed in parallel, within worker thread(s). - * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : - * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). - * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, - * compression is performed inside Caller's thread, and all invocations are blocking */ - ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. - * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. - * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced. */ - ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. - * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. - * It helps preserve compression ratio, while each job is compressed in parallel. - * This value is enforced only when nbWorkers >= 1. - * Larger values increase compression ratio, but decrease speed. - * Possible values range from 0 to 9 : - * - 0 means "default" : value will be determined by the library, depending on strategy - * - 1 means "no overlap" - * - 9 means "full overlap", using a full window size. - * Each intermediate rank increases/decreases load size by a factor 2 : - * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default - * default value varies between 6 and 9, depending on strategy */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_c_rsyncable - * ZSTD_c_format - * ZSTD_c_forceMaxWindow - * ZSTD_c_forceAttachDict - * ZSTD_c_literalCompressionMode - * ZSTD_c_targetCBlockSize - * ZSTD_c_srcSizeHint - * ZSTD_c_enableDedicatedDictSearch - * ZSTD_c_stableInBuffer - * ZSTD_c_stableOutBuffer - * ZSTD_c_blockDelimiters - * ZSTD_c_validateSequences - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly; - * also, the enums values themselves are unstable and can still change. - */ - ZSTD_c_experimentalParam1=500, - ZSTD_c_experimentalParam2=10, - ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001, - ZSTD_c_experimentalParam5=1002, - ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, - ZSTD_c_experimentalParam8=1005, - ZSTD_c_experimentalParam9=1006, - ZSTD_c_experimentalParam10=1007, - ZSTD_c_experimentalParam11=1008, - ZSTD_c_experimentalParam12=1009 -} ZSTD_cParameter; - -typedef struct { - size_t error; - int lowerBound; - int upperBound; -} ZSTD_bounds; - -/*! ZSTD_cParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - lower and upper bounds, both inclusive - */ -ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); - -/*! ZSTD_CCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_cParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is generally only possible during frame initialization (before starting compression). - * Exception : when using multi-threading mode (nbWorkers >= 1), - * the following parameters can be updated _during_ compression (within same frame): - * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. - * new parameters will be active for next job only (after a flush()). - * @return : an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtx_setPledgedSrcSize() : - * Total input data size to be compressed as a single frame. - * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. - * This value will also be controlled at end of frame, and trigger an error if not respected. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. - * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. - * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. - * Note 2 : pledgedSrcSize is only valid once, for the next frame. - * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. - * Note 3 : Whenever all input data is provided and consumed in a single round, - * for example with ZSTD_compress2(), - * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - * this value is automatically overridden by srcSize instead. - */ -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); - -typedef enum { - ZSTD_reset_session_only = 1, - ZSTD_reset_parameters = 2, - ZSTD_reset_session_and_parameters = 3 -} ZSTD_ResetDirective; - -/*! ZSTD_CCtx_reset() : - * There are 2 different things that can be reset, independently or jointly : - * - The session : will stop compressing current frame, and make CCtx ready to start a new one. - * Useful after an error, or to interrupt any ongoing compression. - * Any internal data not yet flushed is cancelled. - * Compression parameters and dictionary remain unchanged. - * They will be used to compress next frame. - * Resetting session never fails. - * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. - * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) - * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) - * - Both : similar to resetting the session, followed by resetting parameters. - */ -ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); - -/*! ZSTD_compress2() : - * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. - * ZSTD_compress2() always starts a new frame. - * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - * @return : compressed size written into `dst` (<= `dstCapacity), - * or an error code if it fails (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - - -/*************************************** -* Advanced decompression API -***************************************/ - -/* The advanced API pushes parameters one by one into an existing DCtx context. - * Parameters are sticky, and remain valid for all following frames - * using the same DCtx context. - * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). - * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). - * Therefore, no new decompression function is necessary. - */ - -typedef enum { - - ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which - * the streaming API will refuse to allocate memory buffer - * in order to protect the host from unreasonable memory requirements. - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). - * Special: value 0 means "use default maximum windowLog". */ - - /* note : additional experimental parameters are also available - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_d_format - * ZSTD_d_stableOutBuffer - * ZSTD_d_forceIgnoreChecksum - * ZSTD_d_refMultipleDDicts - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly - */ - ZSTD_d_experimentalParam1=1000, - ZSTD_d_experimentalParam2=1001, - ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 - -} ZSTD_dParameter; - -/*! ZSTD_dParam_getBounds() : - * All parameters must belong to an interval with lower and upper bounds, - * otherwise they will either trigger an error or be automatically clamped. - * @return : a structure, ZSTD_bounds, which contains - * - an error status field, which must be tested using ZSTD_isError() - * - both lower and upper bounds, inclusive - */ -ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); - -/*! ZSTD_DCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_dParameter. - * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). - * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - * Setting a parameter is only possible during frame initialization (before starting decompression). - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); - -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately. - * Parameters can only be reset when no active frame is being decompressed. - * @return : 0, or an error code, which can be tested with ZSTD_isError() - */ -ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); - - -/**************************** -* Streaming -****************************/ - -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - - -/*-*********************************************************************** -* Streaming compression - HowTo -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. -* -* For parallel execution, use one separate ZSTD_CStream per thread. -* -* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. -* -* Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. -* When in doubt, it's recommended to fully initialize the context before usage. -* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), -* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to -* set more specific parameters, the pledged source size, or load a dictionary. -* -* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to -* consume input stream. The function will automatically update both `pos` -* fields within `input` and `output`. -* Note that the function may not consume the entire input, for example, because -* the output buffer is already full, in which case `input.pos < input.size`. -* The caller must check if input has been entirely consumed. -* If not, the caller must make some room to receive more compressed data, -* and then present again remaining input data. -* note: ZSTD_e_continue is guaranteed to make some forward progress when called, -* but doesn't guarantee maximal forward progress. This is especially relevant -* when compressing with multiple threads. The call won't block if it can -* consume some input, but if it can't it will wait for some, but not all, -* output to be flushed. -* @return : provides a minimum amount of data remaining to be flushed from internal buffers -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, -* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. -* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). -* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the -* operation. -* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if internal buffers are entirely flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. -* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to -* start a new frame. -* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will -* block until the flush is complete or the output buffer is full. -* @return : 0 if frame fully completed and fully flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ - /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ -/*===== ZSTD_CStream management functions =====*/ -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -/*===== Streaming compression functions =====*/ -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush=1, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. - * note : multithreaded compression will block to flush as much output as possible. */ - ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. - * note that frame is only closed after compressed data is fully flushed (return value == 0). - * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). - : note : multithreaded compression will block to flush as much output as possible. */ -} ZSTD_EndDirective; - -/*! ZSTD_compressStream2() : - * Behaves about the same as ZSTD_compressStream, with additional control on end directive. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - * - output->pos must be <= dstCapacity, input->pos must be <= srcSize - * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - endOp must be a valid directive - * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/* These buffer sizes are softly recommended. - * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. - * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), - * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. - * - * However, note that these recommendations are from the perspective of a C caller program. - * If the streaming interface is invoked from some other language, - * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, - * a major performance rule is to reduce crossing such interface to an absolute minimum. - * It's not rare that performance ends being spent more into the interface, rather than compression itself. - * In which cases, prefer using large buffers, as large as practical, - * for both input and output, to reduce the nb of roundtrips. - */ -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ - - -/* ***************************************************************************** - * This following is a legacy streaming API. - * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). - * It is redundant, but remains fully supported. - * Advanced parameters and dictionary compression can only be used through the - * new API. - ******************************************************************************/ - -/*! - * Equivalent to: - * - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - */ -ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/*! - * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). - * NOTE: The return value is different. ZSTD_compressStream() returns a hint for - * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the minimum nb of bytes left to flush (if non-zero and not an error). - */ -ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ -ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ -ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); - - -/*-*************************************************************************** -* Streaming decompression - HowTo -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation. -* @return : recommended first input size -* Alternatively, use advanced API to set specific properties. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos` fields. -* If `input.pos < input.size`, some input has not been consumed. -* It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, respecting output buffer size. -* If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., -* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. -* @return : 0 when a frame is completely decoded and fully flushed, -* or an error code, which can be tested using ZSTD_isError(), -* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. -* *******************************************************************************/ - -typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ - /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ -/*===== ZSTD_DStream management functions =====*/ -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -/*===== Streaming decompression functions =====*/ - -/* This function is redundant with the advanced API and equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, NULL); - */ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); - -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - -ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ - - -/************************** -* Simple dictionary API -***************************/ -/*! ZSTD_compress_usingDict() : - * Compression at an explicit compression level using a Dictionary. - * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); - -/*! ZSTD_decompress_usingDict() : - * Decompression using a known Dictionary. - * Dictionary must be identical to the one used during compression. - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*********************************** - * Bulk processing dictionary API - **********************************/ -typedef struct ZSTD_CDict_s ZSTD_CDict; - -/*! ZSTD_createCDict() : - * When compressing multiple messages or blocks using the same dictionary, - * it's recommended to digest the dictionary only once, since it's a costly operation. - * ZSTD_createCDict() will create a state from digesting a dictionary. - * The resulting state can be used for future compression operations with very limited startup cost. - * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. - * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, - * in which case the only thing that it transports is the @compressionLevel. - * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, - * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); - -/*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); - -/*! ZSTD_compress_usingCDict() : - * Compression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. - * Note : compression level is _decided at dictionary creation time_, - * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); - - -typedef struct ZSTD_DDict_s ZSTD_DDict; - -/*! ZSTD_createDDict() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); - -/*! ZSTD_decompress_usingDDict() : - * Decompression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); - - -/******************************** - * Dictionary helper functions - *******************************/ - -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); - -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); - - -/******************************************************************************* - * Advanced dictionary and prefix API - * - * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. - ******************************************************************************/ - - -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - * Note 2 : Loading a dictionary involves building tables. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Tables are dependent on compression parameters, and for this reason, - * compression parameters can no longer be changed after loading a dictionary. - * Note 3 :`dict` content will be copied internally. - * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. - * Note that compression parameters are enforced from within CDict, - * and supersede any compression parameter previously set within CCtx. - * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. - * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. - * The dictionary will remain valid for future compressed frames using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Referencing a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); - -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compressed frame. - * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression. - * Its content must remain unmodified during compression. - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_c_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). - * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); - -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Loading a dictionary involves building tables, - * which has a non-negligible impact on CPU usage and latency. - * It's recommended to "load once, use many times", to amortize the cost - * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. - * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. - * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of - * how dictionary content is loaded and interpreted. - */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_refDDict() : - * Reference a prepared dictionary, to be used to decompress next frames. - * The dictionary remains active for decompression of future frames using same DCtx. - * - * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function - * will store the DDict references in a table, and the DDict used for decompression - * will be determined at decompression time, as per the dict ID in the frame. - * The memory for the table is allocated on the first call to refDDict, and can be - * freed with ZSTD_freeDCtx(). - * - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Special: referencing a NULL DDict means "return to no-dictionary mode". - * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -/*! ZSTD_DCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) to decompress next frame. - * This is the reverse operation of ZSTD_CCtx_refPrefix(), - * and must use the same prefix as the one used during compression. - * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_decompressStream() returns 0. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary - * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. - * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) - * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A full dictionary is more costly, as it requires building tables. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize); - -/* === Memory management === */ - -/*! ZSTD_sizeof_*() : - * These functions give the _current_ memory usage of selected object. - * Note that object memory usage can evolve (increase or decrease) over time. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); - -#endif /* ZSTD_H_235446 */ - - -/* ************************************************************************************** - * ADVANCED AND EXPERIMENTAL FUNCTIONS - **************************************************************************************** - * The definitions in the following section are considered experimental. - * They are provided for advanced scenarios. - * They should never be used with a dynamic library, as prototypes may change in the future. - * Use them only in association with static linking. - * ***************************************************************************************/ - -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY - -/**************************************************************************************** - * experimental API (static linking only) - **************************************************************************************** - * The following symbols and constants - * are not planned to join "stable API" status in the near future. - * They can still change in future versions. - * Some of them are planned to remain in the static_only section indefinitely. - * Some of them might be removed in the future (especially when redundant with existing stable functions) - * ***************************************************************************************/ - -#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ -#define ZSTD_SKIPPABLEHEADERSIZE 8 - -/* compression parameter bounds */ -#define ZSTD_WINDOWLOG_MAX_32 30 -#define ZSTD_WINDOWLOG_MAX_64 31 -#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) -#define ZSTD_WINDOWLOG_MIN 10 -#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) -#define ZSTD_HASHLOG_MIN 6 -#define ZSTD_CHAINLOG_MAX_32 29 -#define ZSTD_CHAINLOG_MAX_64 30 -#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) -#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN -#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ -#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ -#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX -#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ -#define ZSTD_STRATEGY_MIN ZSTD_fast -#define ZSTD_STRATEGY_MAX ZSTD_btultra2 - - -#define ZSTD_OVERLAPLOG_MIN 0 -#define ZSTD_OVERLAPLOG_MAX 9 - -#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame - * requiring larger than (1< 0: - * If litLength != 0: - * rep == 1 --> offset == repeat_offset_1 - * rep == 2 --> offset == repeat_offset_2 - * rep == 3 --> offset == repeat_offset_3 - * If litLength == 0: - * rep == 1 --> offset == repeat_offset_2 - * rep == 2 --> offset == repeat_offset_3 - * rep == 3 --> offset == repeat_offset_1 - 1 - * - * Note: This field is optional. ZSTD_generateSequences() will calculate the value of - * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not - * use this 'rep' field at all (as of now). - */ -} ZSTD_Sequence; - -typedef struct { - unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /**< dispatch table : larger == faster, more memory */ - unsigned searchLog; /**< nb of searches : larger == more compression, slower */ - unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ - ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ -} ZSTD_compressionParameters; - -typedef struct { - int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ - int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ - int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ -} ZSTD_frameParameters; - -typedef struct { - ZSTD_compressionParameters cParams; - ZSTD_frameParameters fParams; -} ZSTD_parameters; - -typedef enum { - ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ - ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ - ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ -} ZSTD_dictContentType_e; - -typedef enum { - ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ -} ZSTD_dictLoadMethod_e; - -typedef enum { - ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. - * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring this instruction. */ -} ZSTD_format_e; - -typedef enum { - /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ - ZSTD_d_validateChecksum = 0, - ZSTD_d_ignoreChecksum = 1 -} ZSTD_forceIgnoreChecksum_e; - -typedef enum { - /* Note: this enum controls ZSTD_d_refMultipleDDicts */ - ZSTD_rmd_refSingleDDict = 0, - ZSTD_rmd_refMultipleDDicts = 1 -} ZSTD_refMultipleDDicts_e; - -typedef enum { - /* Note: this enum and the behavior it controls are effectively internal - * implementation details of the compressor. They are expected to continue - * to evolve and should be considered only in the context of extremely - * advanced performance tuning. - * - * Zstd currently supports the use of a CDict in three ways: - * - * - The contents of the CDict can be copied into the working context. This - * means that the compression can search both the dictionary and input - * while operating on a single set of internal tables. This makes - * the compression faster per-byte of input. However, the initial copy of - * the CDict's tables incurs a fixed cost at the beginning of the - * compression. For small compressions (< 8 KB), that copy can dominate - * the cost of the compression. - * - * - The CDict's tables can be used in-place. In this model, compression is - * slower per input byte, because the compressor has to search two sets of - * tables. However, this model incurs no start-up cost (as long as the - * working context's tables can be reused). For small inputs, this can be - * faster than copying the CDict's tables. - * - * - The CDict's tables are not used at all, and instead we use the working - * context alone to reload the dictionary and use params based on the source - * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). - * This method is effective when the dictionary sizes are very small relative - * to the input size, and the input size is fairly large to begin with. - * - * Zstd has a simple internal heuristic that selects which strategy to use - * at the beginning of a compression. However, if experimentation shows that - * Zstd is making poor choices, it is possible to override that choice with - * this enum. - */ - ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ - ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ -} ZSTD_dictAttachPref_e; - -typedef enum { - ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. - * Negative compression levels will be uncompressed, and positive compression - * levels will be compressed. */ - ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be - * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ -} ZSTD_literalCompressionMode_e; - - -/*************************************** -* Frame size functions -***************************************/ - -/*! ZSTD_findDecompressedSize() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - decompressed size of all data in all successive frames - * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - * In which case, it's necessary to use streaming mode to decompress data. - * note 2 : decompressed size is always present when compression is done with ZSTD_compress() - * note 3 : decompressed size can be very large (64-bits value), - * potentially larger than what local system can handle as a single memory segment. - * In which case, it's necessary to use streaming mode to decompress data. - * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - * Always ensure result fits within application's authorized limits. - * Each application can set its own limits. - * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - * read each contained frame header. This is fast as most of the data is skipped, - * however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTD_decompressBound() : - * `src` should point to the start of a series of ZSTD encoded and/or skippable frames - * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary at `src + srcSize`) - * @return : - upper-bound for the decompressed size of all data in all successive frames - * - if an error occurred: ZSTD_CONTENTSIZE_ERROR - * - * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. - * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. - * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. - * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: - * upper-bound = # blocks * min(128 KB, Window_Size) - */ -ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); - -/*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. - * @return : size of the Frame Header, - * or an error code (if srcSize is too small) */ -ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); - -typedef enum { - ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ - ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ -} ZSTD_sequenceFormat_e; - -/*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. - * - * Each block will end with a dummy sequence - * with offset == 0, matchLength == 0, and litLength == length of last literals. - * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) - * simply acts as a block delimiter. - * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters - * @return : number of sequences generated - */ - -ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); - -/*! ZSTD_mergeBlockDelimiters() : - * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. - * - * As such, the final generated result has no explicit representation of block boundaries, - * and the final last literals segment is not represented in the sequences. - * - * The output of this function can be fed into ZSTD_compressSequences() with CCtx - * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters - * @return : number of sequences left after merging - */ -ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); - -/*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. - * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) - * The entire source is compressed into a single frame. - * - * The compression behavior changes based on cctx params. In particular: - * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on - * the block size derived from the cctx, and sequences may be split. This is the default setting. - * - * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain - * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. - * - * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined - * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. - * - * In addition to the two adjustable experimental params, there are other important cctx params. - * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. - * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. - * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset - * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md - * - * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. - * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, - * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); - - -/*! ZSTD_writeSkippableFrame() : - * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. - * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, - * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. - * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. - * - * Returns an error if destination buffer is not large enough, if the source size is not representable - * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). - * - * @return : number of bytes written or a ZSTD error. - */ -ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); - - -/*************************************** -* Memory management -***************************************/ - -/*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * - * ZSTD_estimateCCtxSize() will provide a memory budget large enough - * for any compression level up to selected one. - * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate - * does not include space for a window buffer. - * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. - * The estimate will assume the input may be arbitrarily large, - * which is the worst case. - * - * When srcSize can be bound by a known and rather "small" value, - * this fact can be used to provide a tighter estimation - * because the CCtx compression context will need less memory. - * This tighter estimation can be provided by more advanced functions - * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), - * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). - * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. - * - * Note 2 : only single-threaded compression is supported. - * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. - */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); - -/*! ZSTD_estimateCStreamSize() : - * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CStream size estimation is only correct for single-threaded compression. - * ZSTD_DStream memory budget depends on window Size. - * This information can be passed manually, using ZSTD_estimateDStreamSize, - * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); - * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), - * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ -ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); -ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); - -/*! ZSTD_estimate?DictSize() : - * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). - * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). - * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. - */ -ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); -ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); - -/*! ZSTD_initStatic*() : - * Initialize an object using a pre-allocated fixed-size buffer. - * workspace: The memory area to emplace the object into. - * Provided pointer *must be 8-bytes aligned*. - * Buffer must outlive object. - * workspaceSize: Use ZSTD_estimate*Size() to determine - * how large workspace must be to support target scenario. - * @return : pointer to object (same address as workspace, just different type), - * or NULL if error (size too small, incorrect alignment, etc.) - * Note : zstd will never resize nor malloc() when using a static buffer. - * If the object requires more memory than available, - * zstd will just error out (typically ZSTD_error_memory_allocation). - * Note 2 : there is no corresponding "free" function. - * Since workspace is allocated externally, it must be freed externally too. - * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level - * into its associated cParams. - * Limitation 1 : currently not compatible with internal dictionary creation, triggered by - * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). - * Limitation 2 : static cctx currently not compatible with multi-threading. - * Limitation 3 : static dctx is incompatible with legacy support. - */ -ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ - -ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ - -ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams); - -ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType); - - -/*! Custom memory allocation : - * These prototypes make it possible to pass your own allocation/free functions. - * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. - */ -typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTD_freeFunction) (void* opaque, void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_compressionParameters cParams, - ZSTD_customMem customMem); - -/* ! Thread pool : - * These prototypes make it possible to share a thread pool among multiple compression contexts. - * This can limit resources for applications with multiple threads where each one uses - * a threaded compression mode (via ZSTD_c_nbWorkers parameter). - * ZSTD_createThreadPool creates a new thread pool with a given number of threads. - * Note that the lifetime of such pool must exist while being used. - * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value - * to use an internal thread pool). - * ZSTD_freeThreadPool frees a thread pool. - */ -typedef struct POOL_ctx_s ZSTD_threadPool; -ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); -ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); -ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); - - -/* - * This API is temporary and is expected to change or disappear in the future! - */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - const ZSTD_CCtx_params* cctxParams, - ZSTD_customMem customMem); - -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced( - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem); - - -/*************************************** -* Advanced compression functions -***************************************/ - -/*! ZSTD_createCDict_byReference() : - * Create a digested dictionary for compression - * Dictionary content is just referenced, not duplicated. - * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. - * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); - -/*! ZSTD_getDictID_fromCDict() : - * Provides the dictID of the dictionary loaded into `cdict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); - -/*! ZSTD_getCParams() : - * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. - * `estimatedSrcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_getParams() : - * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. - * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); - -/*! ZSTD_checkCParams() : - * Ensure param values remain within authorized range. - * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); - -/*! ZSTD_adjustCParams() : - * optimize params for a given `srcSize` and `dictSize`. - * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. - * `dictSize` must be `0` when there is no dictionary. - * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. - * This function never fails (wide contract) */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); - -/*! ZSTD_compress_advanced() : - * Note : this function is now DEPRECATED. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); - -/*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. - * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams); - - -/*! ZSTD_CCtx_loadDictionary_byReference() : - * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. - * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_loadDictionary_advanced() : - * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_CCtx_refPrefix_advanced() : - * Same as ZSTD_CCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/* === experimental parameters === */ -/* these parameters can be used with ZSTD_setParameter() - * they are not guaranteed to remain supported in the future */ - - /* Enables rsyncable mode, - * which makes compressed files more rsync friendly - * by adding periodic synchronization points to the compressed data. - * The target average block size is ZSTD_c_jobSize / 2. - * It's possible to modify the job size to increase or decrease - * the granularity of the synchronization point. - * Once the jobSize is smaller than the window size, - * it will result in compression ratio degradation. - * NOTE 1: rsyncable mode only works when multithreading is enabled. - * NOTE 2: rsyncable performs poorly in combination with long range mode, - * since it will decrease the effectiveness of synchronization points, - * though mileage may vary. - * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. - * If the selected compression level is already running significantly slower, - * the overall speed won't be significantly impacted. - */ - #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 - -/* Select a compression format. - * The value must be of type ZSTD_format_e. - * See ZSTD_format_e enum definition for details */ -#define ZSTD_c_format ZSTD_c_experimentalParam2 - -/* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ -#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 - -/* Controls whether the contents of a CDict - * are used in place, or copied into the working context. - * Accepts values from the ZSTD_dictAttachPref_e enum. - * See the comments on that enum for an explanation of the feature. */ -#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 - -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_t enum definition for details. - */ -#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 - -/* Tries to fit compressed block size to be around targetCBlockSize. - * No target when targetCBlockSize == 0. - * There is no guarantee on compressed block size (default:0) */ -#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 - -/* User's best guess of source size. - * Hint is not valid when srcSizeHint == 0. - * There is no guarantee that hint is close to actual source size, - * but compression ratio may regress significantly if guess considerably underestimates */ -#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 - -/* Controls whether the new and experimental "dedicated dictionary search - * structure" can be used. This feature is still rough around the edges, be - * prepared for surprising behavior! - * - * How to use it: - * - * When using a CDict, whether to use this feature or not is controlled at - * CDict creation, and it must be set in a CCtxParams set passed into that - * construction (via ZSTD_createCDict_advanced2()). A compression will then - * use the feature or not based on how the CDict was constructed; the value of - * this param, set in the CCtx, will have no effect. - * - * However, when a dictionary buffer is passed into a CCtx, such as via - * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control - * whether the CDict that is created internally can use the feature or not. - * - * What it does: - * - * Normally, the internal data structures of the CDict are analogous to what - * would be stored in a CCtx after compressing the contents of a dictionary. - * To an approximation, a compression using a dictionary can then use those - * data structures to simply continue what is effectively a streaming - * compression where the simulated compression of the dictionary left off. - * Which is to say, the search structures in the CDict are normally the same - * format as in the CCtx. - * - * It is possible to do better, since the CDict is not like a CCtx: the search - * structures are written once during CDict creation, and then are only read - * after that, while the search structures in the CCtx are both read and - * written as the compression goes along. This means we can choose a search - * structure for the dictionary that is read-optimized. - * - * This feature enables the use of that different structure. - * - * Note that some of the members of the ZSTD_compressionParameters struct have - * different semantics and constraints in the dedicated search structure. It is - * highly recommended that you simply set a compression level in the CCtxParams - * you pass into the CDict creation call, and avoid messing with the cParams - * directly. - * - * Effects: - * - * This will only have any effect when the selected ZSTD_strategy - * implementation supports this feature. Currently, that's limited to - * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. - * - * Note that this means that the CDict tables can no longer be copied into the - * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be - * useable. The dictionary can only be attached or reloaded. - * - * In general, you should expect compression to be faster--sometimes very much - * so--and CDict creation to be slightly slower. Eventually, we will probably - * make this mode the default. - */ -#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 - -/* ZSTD_c_stableInBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. - * - * When this flag is enabled zstd won't allocate an input window buffer, - * because the user guarantees it can reference the ZSTD_inBuffer until - * the frame is complete. But, it will still allocate an output buffer - * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also - * avoid the memcpy() from the input buffer to the input window buffer. - * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * - * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find - * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 - -/* ZSTD_c_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells he compressor that the ZSTD_outBuffer will not be resized between - * calls. Specifically: (out.size - out.pos) will never grow. This gives the - * compressor the freedom to say: If the compressed data doesn't fit in the - * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to - * always decompress directly into the output buffer, instead of decompressing - * into an internal buffer and copying to the output buffer. - * - * When this flag is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer. It will still allocate the - * input window buffer (see ZSTD_c_stableInBuffer). - * - * Zstd will check that (out.size - out.pos) never grows and return an error - * if it does. While not strictly necessary, this should prevent surprises. - */ -#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 - -/* ZSTD_c_blockDelimiters - * Default is 0 == ZSTD_sf_noBlockDelimiters. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * - * Designates whether or not the given array of ZSTD_Sequence contains block delimiters - * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. - * See the definition of ZSTD_Sequence for more specifics. - */ -#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 - -/* ZSTD_c_validateSequences - * Default is 0 == disabled. Set to 1 to enable sequence validation. - * - * For use with sequence compression API: ZSTD_compressSequences(). - * Designates whether or not we validate sequences provided to ZSTD_compressSequences() - * during function execution. - * - * Without validation, providing a sequence that does not conform to the zstd spec will cause - * undefined behavior, and may produce a corrupted block. - * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for - * specifics regarding offset/matchlength requirements) then the function will bail out and - * return an error. - * - */ -#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 - -/*! ZSTD_CCtx_getParameter() : - * Get the requested compression parameter value, selected by enum ZSTD_cParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); - - -/*! ZSTD_CCtx_params : - * Quick howto : - * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). - * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to - * an existing CCtx. - * These parameters will be applied to - * all subsequent frames. - * - ZSTD_compressStream2() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. - * - * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() - * for static allocation of CCtx for single-threaded compression. - */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_reset() : - * Reset params to default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_init() : - * Initializes the compression parameters of cctxParams according to - * compression level. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); - -/*! ZSTD_CCtxParams_init_advanced() : - * Initializes the compression and frame parameters of cctxParams according to - * params. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); - -/*! ZSTD_CCtxParams_setParameter() : - * Similar to ZSTD_CCtx_setParameter. - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using - * ZSTD_CCtx_setParametersUsingCCtxParams(). - * @result : a code representing success or failure (which can be tested with - * ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); - -/*! ZSTD_CCtxParams_getParameter() : - * Similar to ZSTD_CCtx_getParameter. - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); - -/*! ZSTD_CCtx_setParametersUsingCCtxParams() : - * Apply a set of ZSTD_CCtx_params to the compression context. - * This can be done even after compression is started, - * if nbWorkers==0, this will have no impact until a new compression is started. - * if nbWorkers>=1, new parameters will be picked up at next job, - * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); - -/*! ZSTD_compressStream2_simpleArgs() : - * Same as ZSTD_compressStream2(), - * but using only integral types as arguments. - * This variant might be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( - ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos, - ZSTD_EndDirective endOp); - - -/*************************************** -* Advanced decompression functions -***************************************/ - -/*! ZSTD_isFrame() : - * Tells if the content of `buffer` starts with a valid Frame Identifier. - * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. - * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. - * Note 3 : Skippable Frame Identifiers are considered valid. */ -ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * Dictionary content is referenced, and therefore stays in dictBuffer. - * It is important that dictBuffer outlives DDict, - * it must remain read accessible throughout the lifetime of DDict */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_byReference() : - * Same as ZSTD_DCtx_loadDictionary(), - * but references `dict` content instead of copying it into `dctx`. - * This saves memory if `dict` remains around., - * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); - -/*! ZSTD_DCtx_loadDictionary_advanced() : - * Same as ZSTD_DCtx_loadDictionary(), - * but gives direct control over - * how to load the dictionary (by copy ? by reference ?) - * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_refPrefix_advanced() : - * Same as ZSTD_DCtx_refPrefix(), but gives finer control over - * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_DCtx_setMaxWindowSize() : - * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. - * This protects a decoder context from reserving too much memory for itself (potential attack scenario). - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); - -/*! ZSTD_DCtx_getParameter() : - * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, - * and store it into int* value. - * @return : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); - -/* ZSTD_d_format - * experimental parameter, - * allowing selection between ZSTD_format_e input compression formats - */ -#define ZSTD_d_format ZSTD_d_experimentalParam1 -/* ZSTD_d_stableOutBuffer - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable. - * - * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the decompressor, and - * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer - * MUST be large enough to fit the entire decompressed frame. This will be - * checked when the frame content size is known. The data in the ZSTD_outBuffer - * in the range [dst, dst + pos) MUST not be modified during decompression - * or you will get data corruption. - * - * When this flags is enabled zstd won't allocate an output buffer, because - * it can write directly to the ZSTD_outBuffer, but it will still allocate - * an input buffer large enough to fit any compressed block. This will also - * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. - * If you need to avoid the input buffer allocation use the buffer-less - * streaming API. - * - * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using - * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, decompression WILL fail if you violate the preconditions. - * - * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST - * not be modified during decompression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate - * matches. Normally zstd maintains its own buffer for this purpose, but passing - * this flag tells zstd to use the user provided buffer. - */ -#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 - -/* ZSTD_d_forceIgnoreChecksum - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * Tells the decompressor to skip checksum validation during decompression, regardless - * of whether checksumming was specified during compression. This offers some - * slight performance benefits, and may be useful for debugging. - * Param has values of type ZSTD_forceIgnoreChecksum_e - */ -#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 - -/* ZSTD_d_refMultipleDDicts - * Experimental parameter. - * Default is 0 == disabled. Set to 1 to enable - * - * If enabled and dctx is allocated on the heap, then additional memory will be allocated - * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() - * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead - * store all references. At decompression time, the appropriate dictID is selected - * from the set of DDicts based on the dictID in the frame. - * - * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. - * - * Param has values of byte ZSTD_refMultipleDDicts_e - * - * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory - * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. - * Memory is allocated as per ZSTD_DCtx::customMem. - * - * Although this function allocates memory for the table, the user is still responsible for - * memory management of the underlying ZSTD_DDict* themselves. - */ -#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 - - -/*! ZSTD_DCtx_setFormat() : - * Instruct the decoder context about what kind of data to decode next. - * This instruction is mandatory to decode data without a fully-formed header, - * such ZSTD_f_zstd1_magicless for example. - * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); - -/*! ZSTD_decompressStream_simpleArgs() : - * Same as ZSTD_decompressStream(), - * but using only integral types as arguments. - * This can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( - ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos); - - -/******************************************************************** -* Advanced streaming functions -* Warning : most of these functions are now redundant with the Advanced API. -* Once Advanced API reaches "stable" status, -* redundant functions will be deprecated, and then at some point removed. -********************************************************************/ - -/*===== Advanced Streaming compression functions =====*/ - -/*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * pledgedSrcSize must be correct. If it is not known at init time, use - * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, - * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, - int compressionLevel, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * Creates of an internal CDict (incompatible with static CCtx), except if - * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if - * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - int compressionLevel); - -/*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); - * - * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. - * pledgedSrcSize must be correct. - * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize); - -/*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); - -/*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * ZSTD_CCtx_refCDict(zcs, cdict); - * - * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, - const ZSTD_CDict* cdict, - ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize); - -/*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: - * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); - * - * start a new frame, using same parameters from previous frame. - * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. - * Note that zcs must be init at least once before using ZSTD_resetCStream(). - * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. - * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. - * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, - * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); - - -typedef struct { - unsigned long long ingested; /* nb input bytes read and buffered */ - unsigned long long consumed; /* nb input bytes actually compressed */ - unsigned long long produced; /* nb of compressed bytes generated and buffered */ - unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ - unsigned currentJobID; /* MT only : latest started job nb */ - unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ -} ZSTD_frameProgression; - -/* ZSTD_getFrameProgression() : - * tells how much data has been ingested (read from input) - * consumed (input actually compressed) and produced (output) for current frame. - * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. - * Aggregates progression inside active worker threads. - */ -ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); - -/*! ZSTD_toFlushNow() : - * Tell how many bytes are ready to be flushed immediately. - * Useful for multithreading scenarios (nbWorkers >= 1). - * Probe the oldest active job, defined as oldest job not yet entirely flushed, - * and check its output buffer. - * @return : amount of data stored in oldest job and ready to be flushed immediately. - * if @return == 0, it means either : - * + there is no active job (could be checked with ZSTD_frameProgression()), or - * + oldest job is still actively compressing data, - * but everything it has produced has also been flushed so far, - * therefore flush speed is limited by production speed of oldest job - * irrespective of the speed of concurrent (and newer) jobs. - */ -ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); - - -/*===== Advanced Streaming decompression functions =====*/ - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); - * - * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * ZSTD_DCtx_refDDict(zds, ddict); - * - * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); - -/*! - * This function is deprecated, and is equivalent to: - * - * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); - * - * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x - */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); - - -/********************************************************************* -* Buffer-less and synchronous inner streaming functions -* -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. -********************************************************************* */ - -/** - Buffer-less streaming compression (synchronous mode) - - A ZSTD_CCtx object is required to track streaming operations. - Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. - ZSTD_CCtx object can be re-used multiple times within successive compression operations. - - Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() - - Then, consume your input using ZSTD_compressContinue(). - There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. - - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). - ZSTD_compressContinue() doesn't guarantee recover after a failed compression. - - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). - It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) - - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. - In which case, it will "discard" the relevant memory section from its history. - - Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. - It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. -*/ - -/*===== Buffer-less streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/** - Buffer-less streaming decompression (synchronous mode) - - A ZSTD_DCtx object is required to track streaming operations. - Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. - - First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). - Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. - Data fragment must be large enough to ensure successful decoding. - `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. - errorCode, which can be tested using ZSTD_isError(). - - It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, - such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). - Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. - As a consequence, check that values remain within valid application range. - For example, do not allocate memory blindly, check that `windowSize` is within expectation. - Each application can set its own limits, depending on local restrictions. - For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. - - ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. - ZSTD_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference distance. - There are multiple ways to guarantee this condition. - - The most memory efficient way is to use a round buffer of sufficient size. - Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). - In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, - up to the moment there is not enough room left in the buffer to guarantee decoding another full block, - which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. - At which point, decoding can resume from the beginning of the buffer. - Note that already decoded data stored in the buffer should be flushed before being overwritten. - - There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. - - Finally, if you control the compression process, you can also ignore all buffer size rules, - as long as the encoder and decoder progress in "lock-step", - aka use exactly the same buffer sizes, break contiguity at the same place, etc. - - Once buffers are setup, start decompression, with ZSTD_decompressBegin(). - If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). - - Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. - ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. - It can also be an error code, which can be tested with ZSTD_isError(). - - A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). - This information is not required to properly decode a frame. - - == Special case : skippable frames == - - Skippable frames allow integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by decompressor. - The format of skippable frames is as follows : - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. - For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. -*/ - -/*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; - -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ - -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* misc */ -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - - - - -/* ============================ */ -/** Block level API */ -/* ============================ */ - -/*! - Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). - But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTD_createCCtx() and ZSTD_createDCtx() - - It is necessary to init context before starting - + compression : any ZSTD_compressBegin*() variant, including with dictionary - + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB - + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! - ===> In which case, nothing is produced into `dst` ! - + User __must__ test for such outcome and deal directly with uncompressed data - + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. - Doing so would mess up with statistics history, leading to potential data corruption. - + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! - + In case of multiple successive blocks, should some of them be uncompressed, - decoder must be informed of their existence in order to follow proper history. - Use ZSTD_insertBlock() for such a case. -*/ - -/*===== Raw zstd block functions =====*/ -ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - - -#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ - -#if defined (__cplusplus) -} -#endif -/**** ended inlining ../zstd.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: huf.h ****/ -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#endif -/**** start inlining xxhash.h ****/ -/* - * xxHash - Extremely Fast Hash algorithm - * Header File - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - - -/* **************************** -* Definitions -******************************/ -/**** skipping file: zstd_deps.h ****/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful if you want to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module anymore. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/* **************************** -* Simple Hash Functions -******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ - - -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ - -/*! State allocation, compatible with dynamic libraries */ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - - -/* hash streaming */ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - - -/* ************************** -* Utils -****************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ -#endif - -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); - - -/* ************************** -* Canonical representation -****************************/ -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); - -#endif /* XXHASH_H_5627135585666179 */ - - - -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ -#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) -#define XXH_STATIC_H_3543687687345 - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ - - -# ifdef XXH_PRIVATE_API -/**** start inlining xxhash.c ****/ -/* - * xxHash - Fast Hash algorithm - * Copyright (c) 2012-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ - defined(__ICCARM__) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for ZSTD_malloc(), ZSTD_free() */ -#define ZSTD_DEPS_NEED_MALLOC -/**** skipping file: zstd_deps.h ****/ -static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } -static void XXH_free (void* p) { ZSTD_free(p); } -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } - -#ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -#endif -/**** skipping file: xxhash.h ****/ - - -/* ************************************* -* Compiler Specific Options -***************************************/ -/**** skipping file: compiler.h ****/ - - -/* ************************************* -* Basic Types -***************************************/ -/**** skipping file: mem.h ****/ - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; - -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -#if defined(__ICCARM__) -# include -# define XXH_rotl32(x,r) __ROR(x,(32 - r)) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ************************** -* Utils -****************************/ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - - -/* *************************** -* Simple Hash Functions -*****************************/ - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -/* ************************************************** -* Advanced Hash Functions -****************************************************/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - - -/*** Hash feed ***/ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - - -/* **** XXH64 **** */ - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - if (input != NULL) { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - } - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/* ************************** -* Canonical representation -****************************/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} -/**** ended inlining xxhash.c ****/ -# endif - -#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ - - -#if defined (__cplusplus) -} -#endif -/**** ended inlining xxhash.h ****/ - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ---- static assert (debug) --- */ -#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - - -/*-************************************* -* shared macros -***************************************/ -#undef MIN -#undef MAX -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -/** - * Ignore: this is an internal helper. - * - * This is a helper function to help force C99-correctness during compilation. - * Under strict compilation modes, variadic macro arguments can't be empty. - * However, variadic function arguments can be. Using a function therefore lets - * us statically check that at least one (string) argument was passed, - * independent of the compilation flags. - */ -static INLINE_KEYWORD UNUSED_ATTR -void _force_has_format_string(const char *format, ...) { - (void)format; -} - -/** - * Ignore: this is an internal helper. - * - * We want to force this function invocation to be syntactically correct, but - * we don't want to force runtime evaluation of its arguments. - */ -#define _FORCE_HAS_FORMAT_STRING(...) \ - if (0) { \ - _force_has_format_string(__VA_ARGS__); \ - } - -/** - * Return the specified error if the condition evaluates to true. - * - * In debug modes, prints additional information. - * In order to do that (particularly, printing the conditional that failed), - * this can't just wrap RETURN_ERROR(). - */ -#define RETURN_ERROR_IF(cond, err, ...) \ - if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } - -/** - * Unconditionally return the specified error. - * - * In debug modes, prints additional information. - */ -#define RETURN_ERROR(err, ...) \ - do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } while(0); - -/** - * If the provided expression evaluates to an error code, returns that error code. - * - * In debug modes, prints additional information. - */ -#define FORWARD_IF_ERROR(err, ...) \ - do { \ - size_t const err_code = (err); \ - if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ - __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return err_code; \ - } \ - } while(0); - - -/*-************************************* -* Common constants -***************************************/ -#define ZSTD_OPT_NUM (1<<12) - -#define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; - -#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ - -#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; - -#define ZSTD_FRAMECHECKSUMSIZE 4 - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 - -#define Litbits 8 -#define MaxLit ((1<= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); - - if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { - /* Handle short offset copies. */ - do { - COPY8(op, ip) - } while (op < oend); - } else { - assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); - /* Separate out the first COPY16() call because the copy length is - * almost certain to be short, so the branches have different - * probabilities. Since it is almost certain to be short, only do - * one COPY16() in the first call. Then, do two calls per loop since - * at that point it is more likely to have a high trip count. - */ -#ifdef __aarch64__ - do { - COPY16(op, ip); - } - while (op < oend); -#else - ZSTD_copy16(op, ip); - if (16 >= length) return; - op += 16; - ip += 16; - do { - COPY16(op, ip); - COPY16(op, ip); - } - while (op < oend); -#endif - } -} - -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - if (length > 0) { - ZSTD_memcpy(dst, src, length); - } - return length; -} - -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 - -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 - -/* Controls whether the input/output buffer is buffered or stable. */ -typedef enum { - ZSTD_bm_buffered = 0, /* Buffer the input/output */ - ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ -} ZSTD_bufferMode_e; - - -/*-******************************************* -* Private declarations -*********************************************/ -typedef struct seqDef_s { - U32 offset; /* Offset code of the sequence */ - U16 litLength; - U16 matchLength; -} seqDef; - -typedef struct { - seqDef* sequencesStart; - seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; - size_t maxNbSeq; - size_t maxNbLit; - - /* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength - * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment - * the existing value of the litLength or matchLength by 0x10000. - */ - U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */ - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ -} seqStore_t; - -typedef struct { - U32 litLength; - U32 matchLength; -} ZSTD_sequenceLength; - -/** - * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences - * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. - */ -MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) -{ - ZSTD_sequenceLength seqLen; - seqLen.litLength = seq->litLength; - seqLen.matchLength = seq->matchLength + MINMATCH; - if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { - if (seqStore->longLengthID == 1) { - seqLen.litLength += 0xFFFF; - } - if (seqStore->longLengthID == 2) { - seqLen.matchLength += 0xFFFF; - } - } - return seqLen; -} - -/** - * Contains the compressed frame size and an upper-bound for the decompressed frame size. - * Note: before using `compressedSize`, check for errors using ZSTD_isError(). - * similarly, before using `decompressedBound`, check for errors using: - * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` - */ -typedef struct { - size_t compressedSize; - unsigned long long decompressedBound; -} ZSTD_frameSizeInfo; /* decompress & legacy */ - -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ - -/* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); - - -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif - } -} - - -/* ZSTD_invalidateRepCodes() : - * ensures next compression will not use repcodes from previous block. - * Note : only works with regular variant; - * do not use with extDict variant ! */ -void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ - - -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; /* declared here for decompress and fullbench */ - -/*! ZSTD_getcBlockSize() : - * Provides the size of compressed block from block header `src` */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr); - -/*! ZSTD_decodeSeqHeaders() : - * decode sequence header from src */ -/* Used by: decompress, fullbench (does not get its definition from here) */ -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_CCOMMON_H_MODULE */ -/**** ended inlining zstd_internal.h ****/ - - -/*-**************************************** -* Version -******************************************/ -unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } - -const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } - - -/*-**************************************** -* ZSTD Error Management -******************************************/ -#undef ZSTD_isError /* defined within zstd_internal.h */ -/*! ZSTD_isError() : - * tells if a return value is an error code - * symbol is required for external callers */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTD_getErrorName() : - * provides error code string from function result (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } - -/*! ZSTD_getError() : - * convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTD_getErrorString() : - * provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } - - - -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); -} - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; - } - return ZSTD_calloc(1, size); -} - -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) -{ - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } -} -/**** ended inlining common/zstd_common.c ****/ - -/**** start inlining decompress/huf_decompress.c ****/ -/* ****************************************************************** - * huff0 huffman decoder, - * part of Finite State Entropy library - * Copyright (c) 2013-2021, Yann Collet, Facebook, Inc. - * - * You can contact the author at : - * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* ************************************************************** -* Dependencies -****************************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/compiler.h ****/ -/**** skipping file: ../common/bitstream.h ****/ -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** skipping file: ../common/error_private.h ****/ - -/* ************************************************************** -* Macros -****************************************************************/ - -/* These two optional macros force the use one way or another of the two - * Huffman decompression implementations. You can't force in both directions - * at the same time. - */ -#if defined(HUF_FORCE_DECOMPRESS_X1) && \ - defined(HUF_FORCE_DECOMPRESS_X2) -#error "Cannot force the use of the X1 and X2 decoders at the same time!" -#endif - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_isError ERR_isError - - -/* ************************************************************** -* Byte alignment for workSpace management -****************************************************************/ -#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) -#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) - - -/* ************************************************************** -* BMI2 Variant Wrappers -****************************************************************/ -#if DYNAMIC_BMI2 - -#define HUF_DGEN(fn) \ - \ - static size_t fn##_default( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - if (bmi2) { \ - return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ - } - -#else - -#define HUF_DGEN(fn) \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - (void)bmi2; \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } - -#endif - - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; - -static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) -{ - DTableDesc dtd; - ZSTD_memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - - -#ifndef HUF_FORCE_DECOMPRESS_X2 - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */ - -/** - * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at - * a time. - */ -static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { - U64 D4; - if (MEM_isLittleEndian()) { - D4 = symbol + (nbBits << 8); - } else { - D4 = (symbol << 8) + nbBits; - } - D4 *= 0x0001000100010001ULL; - return D4; -} - -typedef struct { - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; - U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; - BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; -} HUF_ReadDTableX1_Workspace; - - -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; - HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; - - DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); - if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); - - DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); - if (HUF_isError(iSize)) return iSize; - - /* Table header */ - { DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Compute symbols and rankStart given rankVal: - * - * rankVal already contains the number of values of each weight. - * - * symbols contains the symbols ordered by weight. First are the rankVal[0] - * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. - * symbols[0] is filled (but unused) to avoid a branch. - * - * rankStart contains the offset where each rank belongs in the DTable. - * rankStart[0] is not filled because there are no entries in the table for - * weight 0. - */ - { - int n; - int nextRankStart = 0; - int const unroll = 4; - int const nLimit = (int)nbSymbols - unroll + 1; - for (n=0; n<(int)tableLog+1; n++) { - U32 const curr = nextRankStart; - nextRankStart += wksp->rankVal[n]; - wksp->rankStart[n] = curr; - } - for (n=0; n < nLimit; n += unroll) { - int u; - for (u=0; u < unroll; ++u) { - size_t const w = wksp->huffWeight[n+u]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); - } - } - for (; n < (int)nbSymbols; ++n) { - size_t const w = wksp->huffWeight[n]; - wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; - } - } - - /* fill DTable - * We fill all entries of each weight in order. - * That way length is a constant for each iteration of the outter loop. - * We can switch based on the length to a different inner loop which is - * optimized for that particular case. - */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; - for (w=1; wrankVal[w]; - int const length = (1 << w) >> 1; - int uStart = rankStart; - BYTE const nbBits = (BYTE)(tableLog + 1 - w); - int s; - int u; - switch (length) { - case 1: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart] = D; - uStart += 1; - } - break; - case 2: - for (s=0; ssymbols[symbol + s]; - D.nbBits = nbBits; - dt[uStart+0] = D; - dt[uStart+1] = D; - uStart += 2; - } - break; - case 4: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - uStart += 4; - } - break; - case 8: - for (s=0; ssymbols[symbol + s], nbBits); - MEM_write64(dt + uStart, D4); - MEM_write64(dt + uStart + 4, D4); - uStart += 8; - } - break; - default: - for (s=0; ssymbols[symbol + s], nbBits); - for (u=0; u < length; u += 16) { - MEM_write64(dt + uStart + u + 0, D4); - MEM_write64(dt + uStart + u + 4, D4); - MEM_write64(dt + uStart + u + 8, D4); - MEM_write64(dt + uStart + u + 12, D4); - } - assert(u == length); - uStart += length; - } - break; - } - symbol += symbolCount; - rankStart += symbolCount * length; - } - } - return iSize; -} - -FORCE_INLINE_TEMPLATE BYTE -HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) - -#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) - -HINT_INLINE size_t -HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_1(p, bitDPtr); - HUF_DECODE_SYMBOLX1_2(p, bitDPtr); - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - } - - /* [0-3] symbols remaining */ - if (MEM_32bits()) - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - - return pEnd-pStart; -} - -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - - HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); - - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -FORCE_INLINE_TEMPLATE size_t -HUF_decompress4X1_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* const olimit = oend - 3; - const void* const dtPtr = DTable + 1; - const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - U32 endSignal = 1; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); - - /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ - for ( ; (endSignal) & (op4 < olimit) ; ) { - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_1(op1, &bitD1); - HUF_DECODE_SYMBOLX1_1(op2, &bitD2); - HUF_DECODE_SYMBOLX1_1(op3, &bitD3); - HUF_DECODE_SYMBOLX1_1(op4, &bitD4); - HUF_DECODE_SYMBOLX1_2(op1, &bitD1); - HUF_DECODE_SYMBOLX1_2(op2, &bitD2); - HUF_DECODE_SYMBOLX1_2(op3, &bitD3); - HUF_DECODE_SYMBOLX1_2(op4, &bitD4); - HUF_DECODE_SYMBOLX1_0(op1, &bitD1); - HUF_DECODE_SYMBOLX1_0(op2, &bitD2); - HUF_DECODE_SYMBOLX1_0(op3, &bitD3); - HUF_DECODE_SYMBOLX1_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; - } - - /* check corruption */ - /* note : should not be necessary : op# advance in lock step, and we control op4. - * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - - -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); - -HUF_DGEN(HUF_decompress1X1_usingDTable_internal) -HUF_DGEN(HUF_decompress4X1_usingDTable_internal) - - - -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} - - -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} - -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); -} - - -#endif /* HUF_FORCE_DECOMPRESS_X2 */ - - -#ifndef HUF_FORCE_DECOMPRESS_X1 - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ - -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; -typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; -typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; - - -/* HUF_fillDTableX2Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX2 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } -} - - -static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUF_DEltX2 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, - const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) -{ - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; - U32 *rankStart; - - rankValCol_t* rankVal; - U32* rankStats; - U32* rankStart0; - sortedSymbol_t* sortedSymbol; - BYTE* weightList; - size_t spaceUsed32 = 0; - - rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; - rankStats = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 1; - rankStart0 = (U32 *)workSpace + spaceUsed32; - spaceUsed32 += HUF_TABLELOG_MAX + 2; - sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); - spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; - weightList = (BYTE *)((U32 *)workSpace + spaceUsed32); - spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; - - if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); - - rankStart = rankStart0 + 1; - ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); - - DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ - if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w> consumed; - } } } } - - HUF_fillDTableX2(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - - -FORCE_INLINE_TEMPLATE U32 -HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -FORCE_INLINE_TEMPLATE U32 -HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - ZSTD_memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); - } } - return 1; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) - -HINT_INLINE size_t -HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, - const HUF_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - -FORCE_INLINE_TEMPLATE size_t -HUF_decompress1X2_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); - - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -FORCE_INLINE_TEMPLATE size_t -HUF_decompress4X2_usingDTable_internal_body( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* const olimit = oend - (sizeof(size_t)-1); - const void* const dtPtr = DTable+1; - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal = 1; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); - CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); - CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); - CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - for ( ; (endSignal) & (op4 < olimit); ) { -#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; - endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; -#else - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = (U32)LIKELY( - (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) - & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); -#endif - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - -HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -HUF_DGEN(HUF_decompress4X2_usingDTable_internal) - -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} - - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} - -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - - -#endif /* HUF_FORCE_DECOMPRESS_X1 */ - - -/* ***********************************/ -/* Universal decompression selectors */ -/* ***********************************/ - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - - -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; -#endif - -/** HUF_selectDecoder() : - * Tells which decoder is likely to decode faster, - * based on a set of pre-computed metrics. - * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . - * Assumption : 0 < dstSize <= 128 KB */ -U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - assert(dstSize > 0); - assert(dstSize <= 128*1024); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dstSize; - (void)cSrcSize; - return 0; -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dstSize; - (void)cSrcSize; - return 1; -#else - /* decoder timing evaluation */ - { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ - return DTime1 < DTime0; - } -#endif -} - - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, - size_t dstSize, const void* cSrc, - size_t cSrcSize, void* workSpace, - size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#endif - } -} - -size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); -#endif - } -} - - -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#endif -} - -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} -#endif - -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); -#endif -} - -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); -#endif - } -} - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif - } -} - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} - -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -#endif -/**** ended inlining decompress/huf_decompress.c ****/ -/**** start inlining decompress/zstd_ddict.c ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* zstd_ddict.c : - * concentrates all logic that needs to know the internals of ZSTD_DDict object */ - -/*-******************************************************* -* Dependencies -*********************************************************/ -/**** skipping file: ../common/zstd_deps.h ****/ -/**** start inlining ../common/cpu.h ****/ -/* - * Copyright (c) 2018-2021, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -9081,6 +4010,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { U32 f7b = 0; U32 f7c = 0; #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16 int reg[4]; __cpuid((int*)reg, 0); { @@ -9096,6 +4026,41 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { f7c = (U32)reg[2]; } } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif #elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) /* The following block like the normal cpuid branch below, but gcc * reserves ebx for use of its pic register so we must specially @@ -9257,15 +4222,14 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { #undef X #endif /* ZSTD_COMMON_CPU_H */ -/**** ended inlining ../common/cpu.h ****/ -/**** skipping file: ../common/mem.h ****/ -#define FSE_STATIC_LINKING_ONLY -/**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY -/**** skipping file: ../common/huf.h ****/ -/**** start inlining zstd_decompress_internal.h ****/ +/**** ended inlining cpu.h ****/ +/**** skipping file: mem.h ****/ +/**** skipping file: debug.h ****/ +/**** skipping file: error_private.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** start inlining ../zstd.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -9274,22 +4238,10301 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { * You may select, at your option, one of the above-listed licenses. */ - -/* zstd_decompress_internal: - * objects and definitions shared within lib/decompress modules */ - - #ifndef ZSTD_DECOMPRESS_INTERNAL_H - #define ZSTD_DECOMPRESS_INTERNAL_H +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + + +/* ====== Dependencies ======*/ +#include /* size_t */ + +/**** skipping file: zstd_errors.h ****/ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#include /* INT_MAX */ +#endif /* ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + + +/*====== Decompression helper functions ======*/ + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * When invoking this method on a skippable frame, it will return 0. + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() (obsolete): + * This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid + * Note 1: this method is called _find*() because it's not enough to read the header, + * it may have to scan through the frame's content, to reach its end. + * Note 2: this method also works with Skippable Frames. In which case, + * it returns the size of the complete skippable frame, + * which is always equal to its content size + 8 bytes for headers. */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only @compressionLevel remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-employed multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining content of the compressed frame. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +#if defined (__cplusplus) +extern "C" { +#endif + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. + * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto), + * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_ParamSwitch_e; +#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */ + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e; +#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */ +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */ + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_FrameHeader; +#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */ + +/*! ZSTD_getFrameHeader() : + * decode Frame Header into `zfhPtr`, or requires larger `srcSize`. + * @return : 0 => header is complete, `zfhPtr` is correctly filled, + * >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize); +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */ + ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */ +} ZSTD_SequenceFormat_e; +#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */ + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsCapacity The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.). + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes + * using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit + * can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation. + * By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10). + * ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction. + * + * If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) and then bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused. + * Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_compressSequencesAndLiterals() : + * This is a variant of ZSTD_compressSequences() which, + * instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), + * aka all the literals, already extracted and laid out into a single continuous buffer. + * This can be useful if the process generating the sequences also happens to generate the buffer of literals, + * thus skipping an extraction + caching stage. + * It's a speed optimization, useful when the right conditions are met, + * but it also features the following limitations: + * - Only supports explicit delimiter mode + * - Currently does not support Sequences validation (so input Sequences are trusted) + * - Not compatible with frame checksum, which must be disabled + * - If any block is incompressible, will fail and return an error + * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. + * - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals. + * @litBufCapacity must be at least 8 bytes larger than @litSize. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t litBufCapacity, + size_t decompressedSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, + * so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer. + * + * The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. + * This can be NULL if the caller is not interested in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); -/*-******************************************************* - * Dependencies - *********************************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ -/**** start inlining ../common/zstd_trace.h ****/ /* - * Copyright (c) 2016-2021, Facebook, Inc. + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences*(). + * Designates whether or not provided sequences are validated within ZSTD_compressSequences*() + * during function execution. + * + * When Sequence validation is disabled (default), Sequences are compressed as-is, + * so they must correct, otherwise it would result in a corruption error. + * + * Sequence validation adds some protection, by ensuring that all values respect boundary conditions. + * If a Sequence is detected invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_blockSplitterLevel + * note: this parameter only influences the first splitter stage, + * which is active before producing the sequences. + * ZSTD_c_splitAfterSequences controls the next splitter stage, + * which is active after sequence production. + * Note that both can be combined. + * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included. + * 0 means "auto", which will select a value depending on current ZSTD_c_strategy. + * 1 means no splitting. + * Then, values from 2 to 6 are sorted in increasing cpu load order. + * + * Note that currently the first block is never split, + * to ensure expansion guarantees in presence of incompressible data. + */ +#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6 +#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20 + +/* ZSTD_c_splitAfterSequences + * This is a stronger splitter algorithm, + * based on actual sequences previously produced by the selected parser. + * It's also slower, and as a consequence, mostly used for high compression levels. + * While the post-splitter does overlap with the pre-splitter, + * both can nonetheless be combined, + * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX, + * resulting in higher compression ratio than just one of them. + * + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_ParamSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_repcodeResolution + * This parameter only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters (may change in the future). + * + * This parameter affects how zstd parses external sequences, + * provided via the ZSTD_compressSequences*() API + * or from an external block-level sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets within + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences*() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level (currently: level<10 disables, level>=10 enables). + */ +#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19 +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */ + + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +/**** ended inlining ../zstd.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: fse.h ****/ +/**** skipping file: huf.h ****/ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +/**** start inlining xxhash.h ****/ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Local adaptations for Zstandard */ + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + +#if defined (__cplusplus) +} +#endif + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + +#if defined (__cplusplus) +} /* end of extern "C" */ +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +#if defined (__cplusplus) +extern "C" { +#endif + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#include /* memcmp, memcpy */ +#include /* ULLONG_MAX */ + +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_NO_STDLIB */ + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#if defined (__cplusplus) +} +#endif + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +#if defined (__cplusplus) +} /* extern "C" */ +#endif +# endif +# endif /* XXH_VSX_BE */ + +#if defined (__cplusplus) +extern "C" { +#endif +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ + +#if defined (__cplusplus) +} /* extern "C" */ +#endif + +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + +#if defined (__cplusplus) +extern "C" { +#endif +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n -/* weak symbol support */ -#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && defined(__GNUC__) && \ +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386, aarch64 and risc-v. + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ - !defined(__CYGWIN__) + !defined(__CYGWIN__) && !defined(_AIX) # define ZSTD_HAVE_WEAK_SYMBOLS 1 #else # define ZSTD_HAVE_WEAK_SYMBOLS 0 @@ -9345,7 +14594,7 @@ typedef struct { /** * Non-zero if streaming (de)compression is used. */ - unsigned streaming; + int streaming; /** * The dictionary ID. */ @@ -9354,7 +14603,7 @@ typedef struct { * Is the dictionary cold? * Only set on decompression. */ - unsigned dictionaryIsCold; + int dictionaryIsCold; /** * The dictionary size or zero if no dictionary. */ @@ -9404,14 +14653,15 @@ typedef unsigned long long ZSTD_TraceCtx; * @returns Non-zero if tracing is enabled. The return value is * passed to ZSTD_trace_compress_end(). */ -ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx); +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); /** * Trace the end of a compression call. * @param ctx The return value of ZSTD_trace_compress_begin(). * @param trace The zstd tracing info. */ -void ZSTD_trace_compress_end( +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( ZSTD_TraceCtx ctx, ZSTD_Trace const* trace); @@ -9422,25 +14672,2392 @@ void ZSTD_trace_compress_end( * @returns Non-zero if tracing is enabled. The return value is * passed to ZSTD_trace_compress_end(). */ -ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx); +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); /** * Trace the end of a decompression call. * @param ctx The return value of ZSTD_trace_decompress_begin(). * @param trace The zstd tracing info. */ -void ZSTD_trace_decompress_end( +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( ZSTD_TraceCtx ctx, ZSTD_Trace const* trace); #endif /* ZSTD_TRACE */ -#if defined (__cplusplus) +#endif /* ZSTD_TRACE_H */ +/**** ended inlining zstd_trace.h ****/ +#else +# define ZSTD_TRACE 0 +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#endif /* ZSTD_CCOMMON_H_MODULE */ +/**** ended inlining zstd_internal.h ****/ + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } + +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +/**** ended inlining common/zstd_common.c ****/ + +/**** start inlining decompress/huf_decompress.c ****/ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/compiler.h ****/ +/**** skipping file: ../common/bitstream.h ****/ +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ +/**** skipping file: ../common/bits.h ****/ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 +#endif + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const istart = (const BYTE*)src; + + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } #endif -#endif /* ZSTD_TRACE_H */ -/**** ended inlining ../common/zstd_trace.h ****/ +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } + +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} +/**** ended inlining decompress/huf_decompress.c ****/ +/**** start inlining decompress/zstd_ddict.c ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +/**** start inlining ../common/allocations.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +/**** skipping file: zstd_deps.h ****/ + +/**** skipping file: compiler.h ****/ +#define ZSTD_STATIC_LINKING_ONLY +/**** skipping file: ../zstd.h ****/ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ +/**** ended inlining ../common/allocations.h ****/ +/**** skipping file: ../common/zstd_deps.h ****/ +/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/mem.h ****/ +#define FSE_STATIC_LINKING_ONLY +/**** skipping file: ../common/fse.h ****/ +/**** skipping file: ../common/huf.h ****/ +/**** start inlining zstd_decompress_internal.h ****/ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +/**** skipping file: ../common/mem.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ @@ -9460,7 +17077,7 @@ static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; -static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -9495,12 +17112,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = { #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef struct { ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ U32 rep[ZSTD_REP_NUM]; U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; } ZSTD_entropyDTables_t; @@ -9526,6 +17144,22 @@ typedef struct { size_t ddictPtrCount; } ZSTD_DDictHashSet; +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -9539,7 +17173,7 @@ struct ZSTD_DCtx_s const void* virtualStart; /* virtual start of previous segment if it was just before current one */ const void* dictEnd; /* end of previous segment */ size_t expected; - ZSTD_frameHeader fParams; + ZSTD_FrameHeader fParams; U64 processedCSize; U64 decodedSize; blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ @@ -9556,7 +17190,10 @@ struct ZSTD_DCtx_s size_t litSize; size_t rleSize; size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif /* dictionary */ ZSTD_DDict* ddictLocal; @@ -9566,6 +17203,8 @@ struct ZSTD_DCtx_s ZSTD_dictUses_e dictUses; ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; /* streaming */ ZSTD_dStreamStage streamStage; @@ -9578,16 +17217,21 @@ struct ZSTD_DCtx_s size_t outStart; size_t outEnd; size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; +#endif U32 hostageByte; int noForwardProgress; ZSTD_bufferMode_e outBufferMode; ZSTD_outBuffer expectedOutBuffer; /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t oversizedDuration; @@ -9603,6 +17247,14 @@ struct ZSTD_DCtx_s #endif }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} /*-******************************************************* * Shared internal functions @@ -9626,7 +17278,7 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); /**** ended inlining zstd_decompress_internal.h ****/ /**** start inlining zstd_ddict.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -9672,1373 +17324,7 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /**** ended inlining zstd_ddict.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** start inlining ../legacy/zstd_legacy.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_LEGACY_H -#define ZSTD_LEGACY_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -/**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/error_private.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ - -#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) -# undef ZSTD_LEGACY_SUPPORT -# define ZSTD_LEGACY_SUPPORT 8 -#endif - -#if (ZSTD_LEGACY_SUPPORT <= 1) -/**** start inlining zstd_v01.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V01_H_28739879432 -#define ZSTD_V01_H_28739879432 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error -*/ -unsigned ZSTDv01_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; -ZSTDv01_Dctx* ZSTDv01_createDCtx(void); -size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); - -size_t ZSTDv01_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); - -size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); -size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ -#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V01_H_28739879432 */ -/**** ended inlining zstd_v01.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) -/**** start inlining zstd_v02.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V02_H_4174539423 -#define ZSTD_V02_H_4174539423 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error -*/ -unsigned ZSTDv02_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx; -ZSTDv02_Dctx* ZSTDv02_createDCtx(void); -size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx); - -size_t ZSTDv02_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx); - -size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx); -size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V02_H_4174539423 */ -/**** ended inlining zstd_v02.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) -/**** start inlining zstd_v03.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V03_H_298734209782 -#define ZSTD_V03_H_298734209782 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - - /** -ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error -*/ -unsigned ZSTDv03_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx; -ZSTDv03_Dctx* ZSTDv03_createDCtx(void); -size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx); - -size_t ZSTDv03_decompressDCtx(void* ctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - -/* ************************************* -* Streaming functions -***************************************/ -size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx); - -size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx); -size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V03_H_298734209782 */ -/**** ended inlining zstd_v03.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) -/**** start inlining zstd_v04.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTD_V04_H_91868324769238 -#define ZSTD_V04_H_91868324769238 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* ************************************* -* Includes -***************************************/ -#include /* size_t */ - - -/* ************************************* -* Simple one-step function -***************************************/ -/** -ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format - compressedSize : is the exact source size - maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. - It must be equal or larger than originalSize, otherwise decompression will fail. - return : the number of bytes decompressed into destination buffer (originalSize) - or an errorCode if it fails (which can be tested using ZSTDv01_isError()) -*/ -size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - /** - ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ - void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/** -ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error -*/ -unsigned ZSTDv04_isError(size_t code); - - -/* ************************************* -* Advanced functions -***************************************/ -typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx; -ZSTDv04_Dctx* ZSTDv04_createDCtx(void); -size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx); - -size_t ZSTDv04_decompressDCtx(ZSTDv04_Dctx* dctx, - void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); - - -/* ************************************* -* Direct Streaming -***************************************/ -size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx); - -size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx); -size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -/** - Use above functions alternatively. - ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). - ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. - Result is the number of bytes regenerated within 'dst'. - It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. -*/ - - -/* ************************************* -* Buffered Streaming -***************************************/ -typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx; -ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void); -size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx); - -size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx); -size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize); - -size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); - -/** ************************************************ -* Streaming decompression -* -* A ZBUFF_DCtx object is required to track streaming operation. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. -* -* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() -* It must be the same content as the one set during compression phase. -* Dictionary content must remain accessible during the decompression process. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ -unsigned ZBUFFv04_isError(size_t errorCode); -const char* ZBUFFv04_getErrorName(size_t errorCode); - - -/** The below functions provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFFv04_recommendedDInSize(void); -size_t ZBUFFv04_recommendedDOutSize(void); - - -/* ************************************* -* Prefix - version detection -***************************************/ -#define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_V04_H_91868324769238 */ -/**** ended inlining zstd_v04.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) -/**** start inlining zstd_v05.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv05_H -#define ZSTDv05_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-************************************* -* Dependencies -***************************************/ -#include /* size_t */ -/**** skipping file: ../common/mem.h ****/ - - -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv05_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ -size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - - /** - ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. - */ -void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/* ************************************* -* Helper functions -***************************************/ -/* Error Management */ -unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */ - - -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; -ZSTDv05_DCtx* ZSTDv05_createDCtx(void); -size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv05_decompressDCtx() : -* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ -size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Simple Dictionary API -*************************/ -/*! ZSTDv05_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ -size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - -/*-************************ -* Advanced Streaming API -***************************/ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; -typedef struct { - U64 srcSize; - U32 windowLog; /* the only useful information to retrieve */ - U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; -} ZSTDv05_parameters; -size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); - -size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); -void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); -size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); -size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* ZBUFF API -*************************/ -typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; -ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); -size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); - -size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); -size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); - -size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression -* -* A ZBUFFv05_DCtx object is required to track streaming operations. -* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources. -* Use ZBUFFv05_decompressInit() to start a new decompression operation, -* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv05_DCtx objects can be reused multiple times. -* -* Use ZBUFFv05_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded -* or an error code, which can be tested using ZBUFFv05_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize() -* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -unsigned ZBUFFv05_isError(size_t errorCode); -const char* ZBUFFv05_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, and tend to offer better latency */ -size_t ZBUFFv05_recommendedDInSize(void); -size_t ZBUFFv05_recommendedDOutSize(void); - - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */ - - - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv0505_H */ -/**** ended inlining zstd_v05.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) -/**** start inlining zstd_v06.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv06_H -#define ZSTDv06_H - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv06_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) -# define ZSTDLIBv06_API __declspec(dllexport) -#else -# define ZSTDLIBv06_API -#endif - - -/* ************************************* -* Simple functions -***************************************/ -/*! ZSTDv06_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/** -ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/* ************************************* -* Helper functions -***************************************/ -ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ - -/* Error Management */ -ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ - - -/* ************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; -ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); -ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv06_decompressDCtx() : -* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ -ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTDv06_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ -ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************ -* Advanced Streaming API -***************************/ -struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; -typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; - -ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); - -ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - - -/* ************************************* -* ZBUFF API -***************************************/ - -typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; -ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); -ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); -ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv06_DCtx object is required to track streaming operations. -* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources. -* Use ZBUFFv06_decompressInit() to start a new decompression operation, -* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv06_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv06_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv06_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize() -* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv06_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode); -ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void); -ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */ - - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv06_BUFFERED_H */ -/**** ended inlining zstd_v06.h ****/ -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) -/**** start inlining zstd_v07.h ****/ -/* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef ZSTDv07_H_235446 -#define ZSTDv07_H_235446 - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIBv07_API __declspec(dllexport) -#else -# define ZSTDLIBv07_API -#endif - - -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv07_getDecompressedSize() : -* @return : decompressed size if known, 0 otherwise. - note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. - note 2 : decompressed size could be wrong or intentionally modified ! - always ensure results fit within application's authorized limits */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv07_decompress() : - `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/** -ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format - srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' - cSize (output parameter) : the number of bytes that would be read to decompress this frame - or an error code if it fails (which can be tested using ZSTDv01_isError()) - dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame - or ZSTD_CONTENTSIZE_ERROR if an error occurs - - note : assumes `cSize` and `dBound` are _not_ NULL. -*/ -void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize, - size_t* cSize, unsigned long long* dBound); - -/*====== Helper functions ======*/ -ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ - - -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv07_decompressDCtx() : -* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv07_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************** -* Advanced Dictionary API -****************************/ -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); - -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv07_frameParams; - -ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ - - - - -/* ************************************* -* Streaming functions -***************************************/ -typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); - -ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void); - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv07_H_235446 */ -/**** ended inlining zstd_v07.h ****/ -#endif - -/** ZSTD_isLegacy() : - @return : > 0 if supported by legacy decoder. 0 otherwise. - return value is the version. -*/ -MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) -{ - U32 magicNumberLE; - if (srcSize<4) return 0; - magicNumberLE = MEM_readLE32(src); - switch(magicNumberLE) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case ZSTDv01_magicNumberLE:return 1; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case ZSTDv02_magicNumber : return 2; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case ZSTDv03_magicNumber : return 3; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case ZSTDv04_magicNumber : return 4; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case ZSTDv05_MAGICNUMBER : return 5; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case ZSTDv06_MAGICNUMBER : return 6; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case ZSTDv07_MAGICNUMBER : return 7; -#endif - default : return 0; - } -} - - -MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) -{ - U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ -#if (ZSTD_LEGACY_SUPPORT <= 5) - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } -#endif - return 0; /* should not be possible */ -} - - -MEM_STATIC size_t ZSTD_decompressLegacy( - void* dst, size_t dstCapacity, - const void* src, size_t compressedSize, - const void* dict,size_t dictSize) -{ - U32 const version = ZSTD_isLegacy(src, compressedSize); - (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { size_t result; - ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv05_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { size_t result; - ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv06_freeDCtx(zd); - return result; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { size_t result; - ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv07_freeDCtx(zd); - return result; - } -#endif - default : - return ERROR(prefix_unknown); - } -} - -MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo; - U32 const version = ZSTD_isLegacy(src, srcSize); - switch(version) - { -#if (ZSTD_LEGACY_SUPPORT <= 1) - case 1 : - ZSTDv01_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 2) - case 2 : - ZSTDv02_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 3) - case 3 : - ZSTDv03_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - ZSTDv04_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - ZSTDv05_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - ZSTDv06_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - ZSTDv07_findFrameSizeInfoLegacy(src, srcSize, - &frameSizeInfo.compressedSize, - &frameSizeInfo.decompressedBound); - break; -#endif - default : - frameSizeInfo.compressedSize = ERROR(prefix_unknown); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - break; - } - if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) { - frameSizeInfo.compressedSize = ERROR(srcSize_wrong); - frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; - } - return frameSizeInfo; -} - -MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, size_t srcSize) -{ - ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize); - return frameSizeInfo.compressedSize; -} - -MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) -{ - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); -#endif - } -} - - -MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, - const void* dict, size_t dictSize) -{ - DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion); - if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); - switch(newVersion) - { - default : - case 1 : - case 2 : - case 3 : - (void)dict; (void)dictSize; - return 0; -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv04_decompressInit(dctx); - ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; - if (dctx==NULL) return ERROR(memory_allocation); - ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); - *legacyContext = dctx; - return 0; - } -#endif - } -} - - - -MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, - ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version); - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - (void)legacyContext; (void)output; (void)input; - return ERROR(version_unsupported); -#if (ZSTD_LEGACY_SUPPORT <= 4) - case 4 : - { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 5) - case 5 : - { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 6) - case 6 : - { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif -#if (ZSTD_LEGACY_SUPPORT <= 7) - case 7 : - { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; - const void* src = (const char*)input->src + input->pos; - size_t readSize = input->size - input->pos; - void* dst = (char*)output->dst + output->pos; - size_t decodedSize = output->size - output->pos; - size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); - output->pos += decodedSize; - input->pos += readSize; - return hintSize; - } -#endif - } -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_LEGACY_H */ -/**** ended inlining ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -11147,7 +17433,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); @@ -11253,12 +17539,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); + return ddict->dictID; } /**** ended inlining decompress/zstd_ddict.c ****/ /**** start inlining decompress/zstd_decompress.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11315,20 +17601,20 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) * Dependencies *********************************************************/ /**** skipping file: ../common/zstd_deps.h ****/ -/**** skipping file: ../common/cpu.h ****/ +/**** skipping file: ../common/allocations.h ****/ +/**** skipping file: ../common/error_private.h ****/ +/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: ../common/mem.h ****/ -/**** skipping file: ../common/zstd_trace.h ****/ +/**** skipping file: ../common/bits.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/xxhash.h ****/ -/**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** start inlining zstd_decompress_block.h ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11362,6 +17648,12 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + /* ZSTD_decompressBlock_internal() : * decompress block, starting at `src`, * into destination buffer `dst`. @@ -11370,7 +17662,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) */ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame); + const void* src, size_t srcSize, const streaming_operation streaming); /* ZSTD_buildFSETable() : * generate FSE decoding table for one symbol (ll, ml or off) @@ -11383,16 +17675,21 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ /**** ended inlining zstd_decompress_block.h ****/ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -/**** skipping file: ../legacy/zstd_legacy.h ****/ +#error Using excluded file: ../legacy/zstd_legacy.h (re-amalgamate source to fix) #endif @@ -11402,11 +17699,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, *************************************/ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 -#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. - * Currently, that means a 0.75 load factor. - * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded - * the load factor of the ddict hash set. - */ +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ #define DDICT_HASHSET_TABLE_BASE_SIZE 64 #define DDICT_HASHSET_RESIZE_FACTOR 2 @@ -11500,12 +17797,15 @@ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); - ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; - ret->ddictPtrCount = 0; - if (!ret || !ret->ddictPtrTable) { + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); return NULL; } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; return ret; } @@ -11564,6 +17864,8 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) @@ -11578,11 +17880,16 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; +#endif dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; - dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION @@ -11603,8 +17910,7 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) return dctx; } -ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) -{ +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); @@ -11615,10 +17921,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) } } +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + ZSTD_DCtx* ZSTD_createDCtx(void) { DEBUGLOG(3, "ZSTD_createDCtx"); - return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } static void ZSTD_clearDict(ZSTD_DCtx* dctx) @@ -11703,6 +18014,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) return 0; } +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -11738,16 +18062,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -11755,8 +18103,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; + zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START; + zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE; + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); return 0; } RETURN_ERROR(prefix_unknown, ""); @@ -11789,7 +18139,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(dictIDSizeCode) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : break; case 1 : dictID = ip[pos]; pos++; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; @@ -11797,7 +18149,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s } switch(fcsID) { - default: assert(0); /* impossible */ + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; case 2 : frameContentSize = MEM_readLE32(ip+pos); break; @@ -11821,12 +18175,11 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize) { return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); } - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -11840,7 +18193,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; } #endif - { ZSTD_frameHeader zfh; + { ZSTD_FrameHeader zfh; if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; if (zfh.frameType == ZSTD_skippableFrame) { @@ -11860,18 +18213,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported, ""); - { - size_t const skippableSize = skippableHeaderSize + sizeU32; + { size_t const skippableSize = skippableHeaderSize + sizeU32; RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + /** ZSTD_findDecompressedSize() : - * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or * skippable frames - * @return : decompressed size of the frames contained */ + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; @@ -11881,9 +18268,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -11891,17 +18276,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) continue; } - { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; } + /* skip to next frame */ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); src = (const BYTE *)src + frameSrcSize; srcSize -= frameSrcSize; @@ -11965,17 +18350,17 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) return frameSizeInfo; } -static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) { ZSTD_frameSizeInfo frameSizeInfo; ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameSizeInfoLegacy(src, srcSize); #endif - if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); assert(ZSTD_isError(frameSizeInfo.compressedSize) || @@ -11986,10 +18371,10 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize const BYTE* const ipstart = ip; size_t remainingSize = srcSize; size_t nbBlocks = 0; - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; /* Extract Frame Header */ - { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); if (ZSTD_isError(ret)) return ZSTD_errorFrameSizeInfo(ret); if (ret > 0) @@ -12023,28 +18408,31 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } + frameSizeInfo.nbBlocks = nbBlocks; frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + /** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ + * See docs in zstd.h + * Note: compatible with legacy mode */ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); - return frameSizeInfo.compressedSize; + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); } /** ZSTD_decompressBound() : * compatible with legacy mode - * `src` must point to the start of a ZSTD frame or a skippeable frame + * `src` must point to the start of a ZSTD frame or a skippable frame * `srcSize` must be at least as large as the frame contained * @return : the maximum decompressed size of the compressed source */ @@ -12053,7 +18441,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long bound = 0; /* Iterate over each frame */ while (srcSize > 0) { - ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); size_t const compressedSize = frameSizeInfo.compressedSize; unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) @@ -12066,6 +18454,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) return bound; } +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_FrameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} /*-************************************************************* * Frame decoding @@ -12091,7 +18521,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -12108,10 +18538,10 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, return regenSize; } -static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming) { #if ZSTD_TRACE - if (dctx->traceCtx) { + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { ZSTD_Trace trace; ZSTD_memset(&trace, 0, sizeof(trace)); trace.version = ZSTD_VERSION_NUMBER; @@ -12167,10 +18597,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; + memset(&blockProperties, 0, sizeof(blockProperties)); // rg [11/30/2025] - added to shut up gcc warning size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); if (ZSTD_isError(cBlockSize)) return cBlockSize; @@ -12178,27 +18614,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: RETURN_ERROR(corruption_detected, "invalid block type"); } - - if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->validateChecksum) + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { XXH64_update(&dctx->xxhState, op, decodedSize); - if (decodedSize != 0) + } + if (decodedSize) /* support dst = NULL,0 */ { op += decodedSize; + } assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; @@ -12222,12 +18679,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); } -static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, @@ -12247,7 +18707,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (ZSTD_isLegacy(src, srcSize)) { + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { size_t decodedSize; size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); if (ZSTD_isError(frameSize)) return frameSize; @@ -12257,6 +18717,15 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + assert(decodedSize <= dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -12268,17 +18737,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, } #endif - { U32 const magicNumber = MEM_readLE32(src); - DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; - continue; + continue; /* check next frame */ } } if (ddict) { @@ -12332,7 +18802,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) switch (dctx->dictUses) { default: assert(0 /* Impossible */); - /* fall-through */ + ZSTD_FALLTHROUGH; case ZSTD_dont_use: ZSTD_clearDict(dctx); return NULL; @@ -12354,7 +18824,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); @@ -12374,8 +18844,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -12388,7 +18858,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t return dctx->expected; if (dctx->bType != bt_raw) return dctx->expected; - return MIN(MAX(inputSize, 1), dctx->expected); + return BOUNDED(1, inputSize, dctx->expected); } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -12396,7 +18866,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { { default: /* should not happen */ assert(0); + ZSTD_FALLTHROUGH; case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader; case ZSTDds_decodeBlockHeader: @@ -12408,6 +18880,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { case ZSTDds_checkChecksum: return ZSTDnit_checksum; case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; case ZSTDds_skipFrame: return ZSTDnit_skippableFrame; } @@ -12491,7 +18964,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : @@ -12560,6 +19034,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; @@ -12572,7 +19047,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -12613,11 +19088,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -12706,7 +19181,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { assert(dctx != NULL); #if ZSTD_TRACE - dctx->traceCtx = ZSTD_trace_decompress_begin(dctx); + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; #endif dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ dctx->stage = ZSTDds_getFrameHeaderSize; @@ -12716,10 +19191,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -12778,7 +19254,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -12787,7 +19263,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; @@ -12816,7 +19292,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, ZSTD_DStream* ZSTD_createDStream(void) { DEBUGLOG(3, "ZSTD_createDStream"); - return ZSTD_createDStream_advanced(ZSTD_defaultCMem); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) @@ -12826,7 +19302,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) { - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_internal(customMem); } size_t ZSTD_freeDStream(ZSTD_DStream* zds) @@ -12894,7 +19370,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -12902,6 +19380,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -12912,6 +19391,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } @@ -12983,6 +19463,15 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + default:; } bounds.error = ERROR(parameter_unsupported); @@ -13023,6 +19512,12 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value case ZSTD_d_refMultipleDDicts: *value = (int)dctx->refMultipleDDicts; return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -13056,6 +19551,14 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value } dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -13067,6 +19570,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { dctx->streamStage = zdss_init; dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { @@ -13083,10 +19587,17 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) return ZSTD_sizeof_DCtx(dctx); } -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) { - size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); - unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, @@ -13094,6 +19605,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long return minRBSize; } +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + size_t ZSTD_estimateDStreamSize(size_t windowSize) { size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); @@ -13105,7 +19621,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize) size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) { U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ - ZSTD_frameHeader zfh; + ZSTD_FrameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; RETURN_ERROR_IF(err>0, srcSize_wrong, ""); @@ -13200,6 +19716,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB U32 someMoreWork = 1; DEBUGLOG(5, "ZSTD_decompressStream"); + assert(zds != NULL); RETURN_ERROR_IF( input->pos > input->size, srcSize_wrong, @@ -13220,10 +19737,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(5, "stage zdss_init => transparent reset "); zds->streamStage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) zds->legacyVersion = 0; +#endif zds->hostageByte = 0; zds->expectedOutBuffer = *output; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); @@ -13240,7 +19759,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); @@ -13272,6 +19790,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -13283,14 +19806,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { - size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; - DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); ip = istart + cSize; - op += decompressedSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ zds->expected = 0; zds->streamStage = zdss_init; someMoreWork = 0; @@ -13309,7 +19833,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); - if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -13325,11 +19850,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered - ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) : 0; ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); @@ -13361,7 +19888,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outBuffSize = neededOutBuffSize; } } } zds->streamStage = zdss_read; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_read: DEBUGLOG(5, "stage zdss_read"); @@ -13374,13 +19901,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; } } if (ip==iend) { someMoreWork = 0; break; } /* no more input */ zds->streamStage = zdss_load; - /* fall-through */ + ZSTD_FALLTHROUGH; case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); @@ -13388,7 +19916,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -13397,8 +19925,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -13408,14 +19939,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -13429,7 +19963,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -13442,8 +19976,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -13480,18 +20014,24 @@ size_t ZSTD_decompressStream_simpleArgs ( void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } /**** ended inlining decompress/zstd_decompress.c ****/ /**** start inlining decompress/zstd_decompress_block.c ****/ /* - * Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13512,12 +20052,12 @@ size_t ZSTD_decompressStream_simpleArgs ( /**** skipping file: ../common/mem.h ****/ #define FSE_STATIC_LINKING_ONLY /**** skipping file: ../common/fse.h ****/ -#define HUF_STATIC_LINKING_ONLY /**** skipping file: ../common/huf.h ****/ /**** skipping file: ../common/zstd_internal.h ****/ /**** skipping file: zstd_decompress_internal.h ****/ /**** skipping file: zstd_ddict.h ****/ /**** skipping file: zstd_decompress_block.h ****/ +/**** skipping file: ../common/bits.h ****/ /*_******************************************************* * Macros @@ -13543,6 +20083,13 @@ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } * Block decoding ***************************************************************/ +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, @@ -13561,36 +20108,90 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, } } +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} -/* Hidden declaration for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); /*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * * @return : nb of bytes read from src (< srcSize ) * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); switch(litEncType) { case set_repeat: DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); - /* fall-through */ + ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13613,8 +20214,15 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -13623,13 +20231,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, dctx->bmi2); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -13637,20 +20246,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), dctx->bmi2); + sizeof(dctx->workspace), flags); } } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); @@ -13658,13 +20276,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case set_basic: { size_t litSize, lhSize; U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13677,27 +20295,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; return lhSize+litSize; } case set_rle: { U32 const lhlCode = ((istart[0]) >> 2) & 3; size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -13706,16 +20339,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -13726,6 +20371,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions @@ -13733,7 +20390,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ + * - prettify output, report below, test with fuzzer to ensure it's correct */ /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<nbBits = 0; cell->nextState = 0; assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->nbAdditionalBits = nbAddBits; cell->baseValue = baseValue; } @@ -13859,7 +20516,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB FORCE_INLINE_TEMPLATE void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; @@ -13922,14 +20579,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -13956,7 +20614,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -13967,10 +20625,10 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; u max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); } *DTablePtr = DTableSpace; @@ -14080,11 +20738,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* SeqHead */ nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); - return 1; - } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); @@ -14097,11 +20750,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, } *nbSeqPtr = nbSeq; + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + /* FSE table descriptors */ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6); + SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3); + SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3); ip++; /* Build DTables */ @@ -14112,7 +20773,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -14124,7 +20785,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -14136,7 +20797,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), - dctx->bmi2); + ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } @@ -14150,7 +20811,6 @@ typedef struct { size_t litLength; size_t matchLength; size_t offset; - const BYTE* match; } seq_t; typedef struct { @@ -14164,9 +20824,6 @@ typedef struct { ZSTD_fseState stateOffb; ZSTD_fseState stateML; size_t prevOffset[ZSTD_REP_NUM]; - const BYTE* prefixStart; - const BYTE* dictEnd; - size_t pos; } seqState_t; /*! ZSTD_overlapCopy8() : @@ -14209,7 +20866,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * The src buffer must be before the dst buffer. */ -static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t const diff = op - ip; BYTE* const oend = op + length; @@ -14225,6 +20882,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ assert(length >= 8); ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; assert(op - ip >= 8); assert(op <= oend); } @@ -14239,12 +20897,35 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ assert(oend > oend_w); ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ip += oend_w - op; - op = oend_w; + op += oend_w - op; } /* Handle the leftovers. */ while (op < oend) *op++ = *ip++; } +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + /* ZSTD_execSequenceEnd(): * This version handles cases that are near the end of the output buffer. It requires * more careful checks to make sure there is no overflow. By separating out these hard @@ -14254,10 +20935,11 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). */ FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -14280,27 +20962,78 @@ size_t ZSTD_execSequenceEnd(BYTE* op, if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart-match); + match = dictEnd - (prefixStart - match); if (match + sequence.matchLength <= dictEnd) { ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -14311,6 +21044,104 @@ size_t ZSTD_execSequence(BYTE* op, assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); /* Handle edge cases in a slow path: * - Read beyond end of literals * - Match end is within WILDCOPY_OVERLIMIT of oend @@ -14320,7 +21151,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -14388,6 +21219,7 @@ size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } + static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) { @@ -14401,24 +21233,14 @@ ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqS } FORCE_INLINE_TEMPLATE void -ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) { - ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) -{ - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; + DStatePtr->state = nextState + lowBits; } /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 * bits before reloading. This value is the maximum number of bytes we read * after reloading when we are decoding long offsets. */ @@ -14428,123 +21250,136 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD : 0) typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; -typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) { seq_t seq; - ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; - ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; - ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; - U32 const llBase = llDInfo.baseValue; - U32 const mlBase = mlDInfo.baseValue; - U32 const ofBase = ofDInfo.baseValue; - BYTE const llBits = llDInfo.nbAdditionalBits; - BYTE const mlBits = mlDInfo.nbAdditionalBits; - BYTE const ofBits = ofDInfo.nbAdditionalBits; - BYTE const totalBits = llBits+mlBits+ofBits; - - /* sequence */ - { size_t offset; - if (ofBits > 1) { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } else { - U32 const ll0 = (llBase == 0); - if (LIKELY((ofBits == 0))) { - if (LIKELY(!ll0)) - offset = seqState->prevOffset[0]; - else { - offset = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } - seq.offset = offset; - } - - seq.matchLength = mlBase; - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - seq.litLength = llBase; - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - - if (prefetch == ZSTD_p_prefetch) { - size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, offset is only used for prefetching */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update - * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). - * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). - * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the - * better option, so it is the default for other compilers. But, if you - * measure that it is worse, please put up a pull request. + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. */ - { -#if defined(__GNUC__) && !defined(__clang__) - const int kUseUpdateFseState = 1; +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); #else - const int kUseUpdateFseState = 0; + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; #endif - if (kUseUpdateFseState) { - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - } else { - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); } } return seq; } -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) { size_t const windowSize = dctx->fParams.windowSize; /* No dictionary used. */ @@ -14558,30 +21393,33 @@ MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefix /* Dictionary is active. */ return 1; } +#endif -MEM_STATIC void ZSTD_assertValidSequence( +static void ZSTD_assertValidSequence( ZSTD_DCtx const* dctx, BYTE const* op, BYTE const* oend, seq_t const seq, BYTE const* prefixStart, BYTE const* virtualStart) { #if DEBUGLEVEL >= 1 - size_t const windowSize = dctx->fParams.windowSize; - size_t const sequenceSize = seq.litLength + seq.matchLength; - BYTE const* const oLitEnd = op + seq.litLength; - DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - assert(op <= oend); - assert((size_t)(oend - op) >= sequenceSize); - assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); - if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { - size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); - /* Offset must be within the dictionary. */ - assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); - assert(seq.offset <= windowSize + dictSize); - } else { - /* Offset must be within our window. */ - assert(seq.offset <= windowSize); + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } } #else (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; @@ -14590,31 +21428,30 @@ MEM_STATIC void ZSTD_assertValidSequence( #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - (void)frame; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - /* Regen sequences */ + /* Literals are split between internal buffer & output buffer */ if (nbSeq) { seqState_t seqState; - size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } RETURN_ERROR_IF( @@ -14630,134 +21467,331 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ #if defined(__GNUC__) && defined(__x86_64__) - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * I've been able to reproduce this issue on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 4"); + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif #endif - for ( ; ; ) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); #endif - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - BIT_reloadDStream(&(seqState.DStream)); - op += oneSeqSize; - /* gcc and clang both don't like early returns in this loop. - * Instead break and check for an error at the end of the loop. - */ - if (UNLIKELY(ZSTD_isError(oneSeqSize))) { - error = oneSeqSize; - break; + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } - if (UNLIKELY(!--nbSeq)) break; } /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (ZSTD_isError(error)) return error; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } - return op-ostart; + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - (void)frame; /* Regen sequences */ if (nbSeq) { -#define STORED_SEQS 4 +#define STORED_SEQS 8 #define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 +#define ADVANCED_SEQS STORED_SEQS seq_t sequences[STORED_SEQS]; int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); seqState_t seqState; int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + dctx->fseEntropy = 1; { int i; for (i=0; ientropy.rep[i]; } - seqState.prefixStart = prefixStart; - seqState.pos = (size_t)(op-prefixStart); - seqState.dictEnd = dictEnd; assert(dst != NULL); assert(iend >= ip); RETURN_ERROR_IF( @@ -14768,37 +21802,95 @@ ZSTD_decompressSequencesLong_body( ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } - RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); #endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } /* save reps for next block */ @@ -14806,25 +21898,34 @@ ZSTD_decompressSequencesLong_body( } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); + ZSTD_memmove(op, litPtr, lastLLSize); op += lastLLSize; } } - return op-ostart; + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -14833,53 +21934,65 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #if DYNAMIC_BMI2 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ #endif /* DYNAMIC_BMI2 */ -typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame); - #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -14894,69 +22007,114 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) total += 1; + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ - - return total; + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) + const void* src, size_t srcSize, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -14964,6 +22122,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. @@ -14971,6 +22146,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -14978,37 +22158,55 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif dctx->ddictIsCold = 0; #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif } } +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ @@ -15020,14 +22218,25 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + dctx->isFrameDecompression = 0; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t dSize; - ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); } /**** ended inlining decompress/zstd_decompress_block.c ****/