diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt index 4d7265145..e4ae29923 100644 --- a/tests/benchmarks/CMakeLists.txt +++ b/tests/benchmarks/CMakeLists.txt @@ -6,8 +6,9 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang)) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3") endif() -# configure Google Benchmark; a fixed release, so that results stay comparable -set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5) +# configure Google Benchmark; a fixed release with a checksum, so that builds are reproducible. Update it like any +# other dependency; comparisons stay fair because both sides are built with the same version (see README.md) +set(JSON_GOOGLE_BENCHMARK_VERSION 1.10.0) include(FetchContent) # only the library is needed; -Werror would break the pinned release as soon as @@ -18,7 +19,7 @@ set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) FetchContent_Declare(benchmark URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz - URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + URL_HASH SHA256=acc5c7c0fbed855c4aaa2d7c3c36fb82b0c15c28f4e151e3f679c2fc0aa404af DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) FetchContent_MakeAvailable(benchmark) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index aaf5abe46..6f067db60 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -27,8 +27,11 @@ The input files are those of [nativejson-benchmark](https://github.com/miloyip/n - Network access on the first configure: CMake downloads Google Benchmark and the [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test data, pass `-DJSON_TestDataDirectory=/test_files`. -- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it, - change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. +- Google Benchmark is fetched as a fixed release with a checksum, so that every build uses the same + code and a download cannot change under you. It is updated like any other dependency when a new release comes out: + change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. Results are kept + comparable by building both sides of a comparison with the same version, not by keeping that version forever (see + [Comparing two versions](#comparing-two-versions)). - The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in `include/`. @@ -110,13 +113,18 @@ Python, unpinned versions work as well. In its output: - `OVERALL_GEOMEAN` summarizes all benchmarks; - `-a` shows only the aggregates, not every repetition. +Both builds use the same Google Benchmark, the one `CMakeLists.txt` names, so only the header differs. For the same +reason, do not compare against result files from an earlier checkout: a different Google Benchmark, compiler or +machine state can shift the numbers as much as the change under test. Rerun the baseline instead. + The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0. Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or build that release's own `tests/benchmarks` against its own header. ## Getting stable numbers -- Build and run both versions on the same machine, one right after the other. +- Build and run both versions on the same machine with the same compiler and Google Benchmark, one right after the + other. - Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in. - On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`. Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core