From acf0cc4ad7eb071a4166f56cd9458c6469ecd0a0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 29 Sep 2026 09:46:21 +0200 Subject: [PATCH] Run the json_view comparison on GitHub runners on demand A workflow runs compare.py with pinned downloads on GitHub-hosted Ubuntu runners and shows the results as the job summary and as an artifact: started by hand (workflow_dispatch: x86-64 or AArch64, GCC or Clang), or when a pull request gets the label "benchmark" (both architectures, GCC). The label trigger gives numbers before the workflow is on the default branch, which workflow_dispatch needs. Shared runners are noisy, so the numbers show where json_view stands on another architecture; published numbers still need a quiet machine. compare.py takes the CPU name from lscpu where /proc/cpuinfo has none (AArch64 Linux), and falls back to the architecture. Checked in Linux containers (AArch64, Clang 15 and GCC 9, offline with the pinned archives). Signed-off-by: Niels Lohmann --- .github/workflows/json_view_benchmarks.yml | 78 ++++++++++++++++++++++ tests/benchmarks/json_view/README.md | 8 +++ tests/benchmarks/json_view/compare.py | 6 +- 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/json_view_benchmarks.yml diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml new file mode 100644 index 000000000..ee41eac9d --- /dev/null +++ b/.github/workflows/json_view_benchmarks.yml @@ -0,0 +1,78 @@ +name: "json_view benchmarks" + +# On demand only: runs the comparison of json_view with yyjson, simdjson, and +# Boost.JSON (tests/benchmarks/json_view/compare.py) on GitHub-hosted runners, +# for numbers from x86-64 and AArch64 Linux. It runs when started by hand, or +# when a pull request gets the label "benchmark" (on both architectures, with +# GCC and the default settings). Shared runners are noisy: the results show +# where json_view stands, but published numbers need a quiet machine (see +# tests/benchmarks/json_view/README.md). + +on: + pull_request: + types: [labeled] + workflow_dispatch: + inputs: + runner: + description: "Runner image" + type: choice + options: + - ubuntu-24.04 + - ubuntu-24.04-arm + default: ubuntu-24.04 + compiler: + description: "Compiler" + type: choice + options: + - g++ + - clang++ + default: g++ + native: + description: "Compile for the runner's CPU (-march=native)" + type: boolean + default: false + rounds: + description: "Rounds of bench_view" + type: number + default: 30 + +permissions: + contents: read + +jobs: + compare: + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'benchmark' + strategy: + matrix: + runner: ${{ fromJSON(github.event_name == 'workflow_dispatch' && format('["{0}"]', inputs.runner) || '["ubuntu-24.04", "ubuntu-24.04-arm"]') }} + runs-on: ${{ matrix.runner }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download test data + run: | + cmake -S . -B build -DJSON_BuildTests=On + cmake --build build --target download_test_data + + - name: Run the comparison + env: + CXX: ${{ inputs.compiler || 'g++' }} + CC: ${{ inputs.compiler == 'clang++' && 'clang' || 'gcc' }} + ROUNDS: ${{ inputs.rounds || 30 }} + NATIVE: ${{ inputs.native && '--native' || '' }} + run: python3 tests/benchmarks/json_view/compare.py --data build/test_files --download --rounds "$ROUNDS" $NATIVE + + - name: Summary + run: cat tests/benchmarks/json_view/results/*.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: json_view-benchmarks-${{ matrix.runner }}-${{ inputs.compiler || 'g++' }} + path: tests/benchmarks/json_view/results/ diff --git a/tests/benchmarks/json_view/README.md b/tests/benchmarks/json_view/README.md index d5e0678c9..f9046c659 100644 --- a/tests/benchmarks/json_view/README.md +++ b/tests/benchmarks/json_view/README.md @@ -29,6 +29,14 @@ python3 tests/benchmarks/json_view/compare.py --data For numbers worth publishing, use a quiet machine (see [Getting stable numbers](../README.md#getting-stable-numbers)), the default 30 rounds or more, and `--native` only if the other libraries were built for the same CPU. +### On GitHub-hosted runners + +The workflow [json_view benchmarks](../../../.github/workflows/json_view_benchmarks.yml) runs `compare.py --download` +on demand: by hand (Actions → "json_view benchmarks" → "Run workflow"), on an x86-64 or AArch64 Ubuntu runner with GCC +or Clang, or when a pull request gets the label `benchmark`, on both architectures with GCC. The results appear as the +job summary and as an artifact. Shared runners are noisy, so these numbers show +where `json_view` stands on another architecture; they are not meant for publication. + ## What is measured `bench_view.cpp` runs four workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a diff --git a/tests/benchmarks/json_view/compare.py b/tests/benchmarks/json_view/compare.py index 544f126e5..0901011ca 100755 --- a/tests/benchmarks/json_view/compare.py +++ b/tests/benchmarks/json_view/compare.py @@ -183,7 +183,11 @@ def cpu_model(): return line.split(':', 1)[1].strip() except OSError: pass - return platform.processor() + # (AArch64 Linux: /proc/cpuinfo has no model name, lscpu knows it) + for line in output(['lscpu']).splitlines(): + if line.startswith('Model name:'): + return line.split(':', 1)[1].strip() + return platform.processor() or platform.machine() def git_commit():