From ec4bdc398a1a0e00ea18abd898d11ff7277ccb50 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 25 Sep 2026 20:43:34 +0200 Subject: [PATCH] Document the benchmarks, make them build and compare versions again, and pin Google Benchmark (#5556) * Document the benchmarks, and make them build and compare versions again The benchmark project hasn't configured since #4793: download_test_data.cmake compiles cmake/detect_libcpp_version.cpp relative to CMAKE_SOURCE_DIR, which is tests/benchmarks when that is the top-level project, so try_run fails and so does `make run_benchmarks`. The path is now relative to the module itself, which is the same file for the main build. The Dump benchmark discarded dump()'s result, which is [[nodiscard]] by now; it warned, and left the optimizer free to shorten the loop. The result is now kept with benchmark::DoNotOptimize. A new cache variable, JSON_BENCHMARK_INCLUDE_DIR, names the directory holding the nlohmann/json.hpp to benchmark (single_include by default, as before), so the same benchmarks can be built against two versions and compared. tests/benchmarks/README.md documents what is measured, how to build and run the benchmarks, how to read the output, and how to compare two versions with Google Benchmark's compare.py; it recommends doing so by hand before a release rather than in CI. Signed-off-by: Niels Lohmann * Point ci_benchmarks at tests/benchmarks The target has configured ${PROJECT_SOURCE_DIR}/benchmarks since it was added in #2561, but the benchmarks live in tests/benchmarks. Signed-off-by: Niels Lohmann * Pin Google Benchmark to release 1.9.5 The benchmarks fetched Google Benchmark's main branch, so two builds on different days could measure with different library code, and CMake 3.30 and later warn that the single-argument FetchContent_Populate() is deprecated. Fetch the 1.9.5 release archive, verified by its SHA-256, with FetchContent_MakeAvailable() instead. That needs CMake 3.14; Google Benchmark itself already needed 3.13. Its -Werror is switched off, so a newer compiler's new warnings cannot break the pinned release, and its install rules are no longer added. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- cmake/ci.cmake | 2 +- cmake/download_test_data.cmake | 2 +- tests/benchmarks/CMakeLists.txt | 36 ++++---- tests/benchmarks/README.md | 130 ++++++++++++++++++++++++++++ tests/benchmarks/src/benchmarks.cpp | 3 +- 5 files changed, 155 insertions(+), 18 deletions(-) create mode 100644 tests/benchmarks/README.md diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 752bdc6f8..573a8e0e0 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -619,7 +619,7 @@ add_custom_target(ci_single_binaries add_custom_target(ci_benchmarks COMMAND ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=Release -GNinja - -S${PROJECT_SOURCE_DIR}/benchmarks -B${PROJECT_BINARY_DIR}/build_benchmarks + -S${PROJECT_SOURCE_DIR}/tests/benchmarks -B${PROJECT_BINARY_DIR}/build_benchmarks COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_benchmarks --target json_benchmarks COMMAND cd ${PROJECT_BINARY_DIR}/build_benchmarks && ./json_benchmarks COMMENT "Run benchmarks" diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index b7211ac54..3b6f9a395 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -77,7 +77,7 @@ if(CMAKE_CROSSCOMPILING) endif() if(NOT DEFINED LIBCPP_VERSION_OUTPUT_CACHED) try_run(RUN_RESULT_VAR COMPILE_RESULT_VAR - "${CMAKE_BINARY_DIR}" SOURCES "${CMAKE_SOURCE_DIR}/cmake/detect_libcpp_version.cpp" + "${CMAKE_BINARY_DIR}" SOURCES "${CMAKE_CURRENT_LIST_DIR}/detect_libcpp_version.cpp" RUN_OUTPUT_VARIABLE LIBCPP_VERSION_OUTPUT COMPILE_OUTPUT_VARIABLE LIBCPP_VERSION_COMPILE_OUTPUT ) diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt index 4aa095e39..4d7265145 100644 --- a/tests/benchmarks/CMakeLists.txt +++ b/tests/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11...3.14) +cmake_minimum_required(VERSION 3.14) project(JSON_Benchmarks LANGUAGES CXX) # set compiler flags @@ -6,29 +6,35 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang)) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3") endif() -# configure Google Benchmarks +# configure Google Benchmark; a fixed release, so that results stay comparable +set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5) include(FetchContent) -FetchContent_Declare( - benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG origin/main - GIT_SHALLOW TRUE -) -FetchContent_GetProperties(benchmark) -if(NOT benchmark_POPULATED) - FetchContent_Populate(benchmark) - set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "" FORCE) - add_subdirectory(${benchmark_SOURCE_DIR} ${benchmark_BINARY_DIR}) -endif() +# only the library is needed; -Werror would break the pinned release as soon as +# a newer compiler adds a warning +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) + +FetchContent_Declare(benchmark + URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz + URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE +) +FetchContent_MakeAvailable(benchmark) # download test data set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake ${CMAKE_MODULE_PATH}) include(download_test_data) +# the header to benchmark; point this at a directory holding another version's +# nlohmann/json.hpp to compare versions (see README.md) +set(JSON_BENCHMARK_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../single_include" CACHE PATH + "directory containing the nlohmann/json.hpp to benchmark") + # benchmark binary add_executable(json_benchmarks src/benchmarks.cpp) target_compile_features(json_benchmarks PRIVATE cxx_std_11) target_link_libraries(json_benchmarks benchmark ${CMAKE_THREAD_LIBS_INIT}) add_dependencies(json_benchmarks download_test_data) -target_include_directories(json_benchmarks PRIVATE ${CMAKE_SOURCE_DIR}/../../single_include ${CMAKE_BINARY_DIR}/include) +target_include_directories(json_benchmarks PRIVATE ${JSON_BENCHMARK_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/include) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md new file mode 100644 index 000000000..aaf5abe46 --- /dev/null +++ b/tests/benchmarks/README.md @@ -0,0 +1,130 @@ +# Benchmarks + +Micro-benchmarks for parsing, serialization and the binary formats, written with +[Google Benchmark](https://github.com/google/benchmark). They are not run by CI; see +[When to run them](#when-to-run-them). + +## What is measured + +| benchmark | what it does | +|---|---| +| `ParseFile`, `ParseString` | parse JSON from a file stream or a string | +| `ParseIndented` | parse the large files re-indented by 4 spaces, for the lexer's whitespace handling | +| `Dump` | serialize, compact (`-`) and indented (`4`) | +| `ToCbor`, `BinaryToCbor` | write CBOR; `BinaryToCbor` writes binary values of growing size | +| `FromMsgpack` | read MessagePack; unchanged over the years, so its numbers stay comparable across releases | +| `FromBinaryBuffer`, `FromBinaryFile` | read CBOR, MessagePack, UBJSON, BJData and BSON from a buffer or a `FILE*` | +| `FromBinaryShape` | read deeply nested, container-heavy and scalar-heavy documents in every binary format | +| `FromCborChunkedString` | read CBOR strings split into indefinite-length chunks | + +The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`, +`citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...). +`bytes_per_second` counts the bytes read or written: the JSON text when parsing, the output when serializing. + +## Requirements + +- CMake 3.14 or later, a C++11 compiler, and Ninja for the `make` target. +- Network access on the first configure: CMake downloads Google Benchmark and the + [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test + data, pass `-DJSON_TestDataDirectory=/test_files`. +- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it, + change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. +- The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in + `include/`. + +GCC and Clang builds use `-O3 -flto -DNDEBUG`. + +## Running them + +From the repository root, this builds everything from scratch in `cmake-build-benchmarks` and runs all benchmarks: + +```sh +make run_benchmarks +``` + +To build once and run selectively: + +```sh +cmake -S tests/benchmarks -B build-benchmarks -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build-benchmarks +build-benchmarks/json_benchmarks --benchmark_filter='ParseString|Dump' +``` + +Useful options of `json_benchmarks`: + +| option | effect | +|---|---| +| `--benchmark_list_tests` | list the benchmarks instead of running them | +| `--benchmark_filter=` | run only the benchmarks whose names match | +| `--benchmark_repetitions=` | run every benchmark `n` times and add mean, median, standard deviation and coefficient of variation | +| `--benchmark_enable_random_interleaving=true` | run the repetitions in random order, which spreads out drifts such as thermal throttling | +| `--benchmark_min_time=s` | run each benchmark at least this long (e.g. `2s`) | +| `--benchmark_out= --benchmark_out_format=json` | also write the results to a file, e.g. for `compare.py` | + +## Reading the output + +Each line shows the wall-clock `Time` and the `CPU` time per iteration, the number of `Iterations` Google Benchmark +chose, and the throughput in `bytes_per_second`. With repetitions, the lines ending in `_median` are the ones to +compare. A `_cv` (coefficient of variation) above a few percent means the machine was too noisy for small +differences to mean anything. + +## Comparing two versions + +To see what a change or a release did, build the same benchmarks twice: once against the header of the version to +compare with, and once against the current one. `JSON_BENCHMARK_INCLUDE_DIR` names the directory holding the +`nlohmann/json.hpp` to benchmark. For example, to compare the current checkout with 3.12.0: + +```sh +# the header of the version to compare with +mkdir -p build-baseline-header/nlohmann +git show v3.12.0:single_include/nlohmann/json.hpp > build-baseline-header/nlohmann/json.hpp + +# the same benchmarks, built against either header +cmake -S tests/benchmarks -B build-baseline -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DJSON_BENCHMARK_INCLUDE_DIR="$PWD/build-baseline-header" +cmake -S tests/benchmarks -B build-current -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build-baseline +cmake --build build-current + +# run both, back to back +build-baseline/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \ + --benchmark_out=build-baseline/results.json --benchmark_out_format=json +build-current/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \ + --benchmark_out=build-current/results.json --benchmark_out_format=json +``` + +Google Benchmark ships a tool to compare the two result files. It needs NumPy and SciPy: + +```sh +python3 -m venv build-venv +build-venv/bin/pip install numpy scipy +build-venv/bin/python build-current/_deps/benchmark-src/tools/compare.py -a benchmarks build-baseline/results.json build-current/results.json +``` + +The tool's own `tools/requirements.txt` pins NumPy and SciPy versions that need Python 3.11 or later; with an older +Python, unpinned versions work as well. In its output: + +- the `Time` and `CPU` columns are relative changes: `-0.35` means 35% faster, `+0.10` means 10% slower; +- `_pvalue` lines report a Mann-Whitney U test of whether the two versions differ. It needs at least 9 + repetitions, and a p-value below 0.05 means the difference is unlikely to be noise; +- `OVERALL_GEOMEAN` summarizes all benchmarks; +- `-a` shows only the aggregates, not every repetition. + +The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0. +Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or +build that release's own `tests/benchmarks` against its own header. + +## Getting stable numbers + +- Build and run both versions on the same machine, one right after the other. +- Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in. +- On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`. + Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core + (`taskset -c 2 ...`) helps as well. +- Use 10 or more repetitions with random interleaving, compare medians, and treat changes within the `_cv` as noise. + +## When to run them + +They are a manual step, not part of CI: shared CI runners vary more between runs than most of the effects measured. +Run the comparison above before a release, comparing the previous release tag with `develop`, and for pull requests +that claim to change performance. diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index f949f3f12..613ca1baa 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -131,7 +131,8 @@ static void Dump(benchmark::State& state, const char* filename, int indent) while (state.KeepRunning()) { - j.dump(indent); + std::string output = j.dump(indent); + benchmark::DoNotOptimize(output); } state.SetBytesProcessed(state.iterations() * j.dump(indent).size());