diff --git a/test/renderdiff/generate.sh b/test/renderdiff/generate.sh new file mode 100755 index 0000000000..6b9f5eb54f --- /dev/null +++ b/test/renderdiff/generate.sh @@ -0,0 +1,59 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/bash + +source `dirname $0`/src/preamble.sh + +function start_render_() { + start_ + if [[ ! "$GITHUB_WORKFLOW" ]]; then + if [ ! -d ${MESA_LIB_DIR} ]; then + bash ${BUILD_COMMON_DIR}/get-mesa.sh + fi + + # Install python deps + python3 -m venv ${VENV_DIR} + source ${VENV_DIR}/bin/activate + + NEEDED_PYTHON_DEPS=("numpy" "tifffile") + for cmd in "${NEEDED_PYTHON_DEPS[@]}"; do + if ! python3 -m pip show -q "${cmd}"; then + python3 -m pip install ${cmd} + fi + done + fi + mkdir -p ${OUTPUT_DIR} + CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer +} + +function end_render_() { + if [[ ! "$GITHUB_WORKFLOW" ]]; then + deactivate # End python virtual env + fi + end_ +} + +# Following steps are taken: +# - Get and build mesa +# - Build gltf_viewer +# - Run a test + +start_render_ && \ + python3 ${RENDERDIFF_TEST_DIR}/src/render.py \ + --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ + --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ + --output_dir=${OUTPUT_DIR} \ + --opengl_lib=${MESA_LIB_DIR} && \ + end_render_ diff --git a/test/renderdiff/src/compare.py b/test/renderdiff/src/compare.py new file mode 100644 index 0000000000..182aef3352 --- /dev/null +++ b/test/renderdiff/src/compare.py @@ -0,0 +1,54 @@ +import glob +import os +import sys +import pprint +import json + +from utils import execute, ArgParseImpl, important_print +from image_diff import same_image +from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING + +def _compare_goldens(base_dir, comparison_dir): + render_results = {} + base_files = glob.glob(os.path.join(base_dir, "./**/*.tif")) + for golden_file in base_files: + base_fname = os.path.abspath(golden_file) + test_case = base_fname.replace(f'{os.path.abspath(base_dir)}/', '') + comp_fname = os.path.abspath(os.path.join(comparison_dir, test_case)) + if not os.path.exists(comp_fname): + print(f'file name not found: {comp_fname}') + render_results[test_case] = RESULT_MISSING + continue + if not same_image(base_fname, comp_fname): + render_results[test_case] = RESULT_FAILED + else: + render_results[test_case] = RESULT_OK + return render_results + +if __name__ == '__main__': + parser = ArgParseImpl() + parser.add_argument('--src', help='Directory of the base of the diff.', required=True) + parser.add_argument('--dest', help='Directory of the comparison of the diff.') + parser.add_argument('--out', help='Directory of output for the result of the diff.') + + args, _ = parser.parse_known_args(sys.argv[1:]) + + dest = args.dest + if not dest: + print('Assume the default renderdiff output folder') + dest = os.path.join(os.getcwd(), './out/renderdiff_tests') + assert os.path.exists(dest), f"Destination folder={dest} does not exist." + + results = _compare_goldens(args.src, dest) + + if args.out: + assert os.path.exists(arg.out), f"Output folder={dest} does not exist." + with open(os.path.join(args.out, "compare_results.json", 'w')) as f: + f.write(json.dumps(results)) + + failed = [f" {k}" for k in results.keys() if results[k] != RESULT_OK] + success_count = len(results) - len(failed) + important_print(f'Successfully compared {success_count} / {len(results)} images' + + ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) + if len(failed) > 0: + exit(1) diff --git a/test/renderdiff/src/golden_manager.py b/test/renderdiff/src/golden_manager.py index 7db4162d6e..e4fda3c34e 100644 --- a/test/renderdiff/src/golden_manager.py +++ b/test/renderdiff/src/golden_manager.py @@ -15,6 +15,7 @@ import os import shutil import re +import sys from utils import execute, ArgParseImpl, mkdir_p @@ -132,12 +133,19 @@ class GoldenManager: rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR) shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True) -# For testing only +# The main entry point will enable download content of a branch to a directory if __name__ == "__main__": + parser = ArgParseImpl() + parser.add_argument('--branch', type=str, help='Branch of the golden repo', default='main') + parser.add_argument('--output', type=str, help='Directory to download to', required=True) + + args, _ = parser.parse_known_args(sys.argv[1:]) + + # prepare goldens working directory + golden_dir = args.output + assert os.path.isdir(golden_dir),\ + f"Output directory {golden_dir} does not exist" + + # Download the golden repo into the current working directory golden_manager = GoldenManager(os.getcwd()) - # golden_manager.source_from_and_commit( - # os.path.join(os.getcwd(), 'out/renderdiff_tests'), - # 'First commit (local)', - # branch='branch-test') - # golden_manager.merge_to_main('branch-test', push_to_remote=True) - # golden_manager.download_to(os.path.join(os.getcwd(), 'tmp/goldens')) + golden_manager.download_to(golden_dir, branch=args.branch) diff --git a/test/renderdiff/src/preamble.sh b/test/renderdiff/src/preamble.sh new file mode 100644 index 0000000000..d81b3733cf --- /dev/null +++ b/test/renderdiff/src/preamble.sh @@ -0,0 +1,45 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/bash + +# Sets up the environment for scripts in test/renderdiff/ + +OUTPUT_DIR="$(pwd)/out/renderdiff_tests" +RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff" +MESA_DIR="$(pwd)/mesa/out/" +VENV_DIR="$(pwd)/venv" +BUILD_COMMON_DIR="$(pwd)/build/common" + +os_name=$(uname -s) +if [[ "$os_name" == "Linux" ]]; then + MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu" +elif [[ "$os_name" == "Darwin" ]]; then + MESA_LIB_DIR="${MESA_DIR}lib" +else + echo "Unsupported platform for renderdiff tests" + exit 1 +fi + +function start_() { + if [[ "$GITHUB_WORKFLOW" ]]; then + set -ex + fi +} + +function end_() { + if [[ "$GITHUB_WORKFLOW" ]]; then + set +ex + fi +} diff --git a/test/renderdiff/src/run.py b/test/renderdiff/src/render.py similarity index 55% rename from test/renderdiff/src/run.py rename to test/renderdiff/src/render.py index 524c0a2f94..88d305b310 100644 --- a/test/renderdiff/src/run.py +++ b/test/renderdiff/src/render.py @@ -18,31 +18,18 @@ import json import glob import shutil -from utils import execute, ArgParseImpl, mkdir_p, mv_f -from parse_test_json import parse_test_config_from_path +from utils import execute, ArgParseImpl, mkdir_p, mv_f, important_print + +import test_config from golden_manager import GoldenManager from image_diff import same_image +from results import RESULT_OK, RESULT_FAILED -def important_print(msg): - lines = msg.split('\n') - max_len = max([len(l) for l in lines]) - print('-' * (max_len + 8)) - for line in lines: - diff = max_len - len(line) - information = f'--- {line} ' + (' ' * diff) + '---' - print(information) - print('-' * (max_len + 8)) - -RESULT_OK = 'ok' -RESULT_FAILED_TO_RENDER = 'failed-to-render' -RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff' -RESULT_FAILED_NO_GOLDEN = 'failed-no-golden' - -def run_test(gltf_viewer, - test_config, - output_dir, - opengl_lib=None, - vk_icd=None): +def _render_test_config(gltf_viewer, + test_config, + output_dir, + opengl_lib=None, + vk_icd=None): assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist" assert os.access(gltf_viewer, os.X_OK) @@ -86,7 +73,7 @@ def run_test(gltf_viewer, mv_f(f'{test.name}0.tif', out_tif_name) mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json') else: - result = RESULT_FAILED_TO_RENDER + result = RESULT_FAILED important_print(f'{test_desc} rendering failed with error={out_code}') results.append({ @@ -96,23 +83,6 @@ def run_test(gltf_viewer, }) return named_output_dir, results -def compare_goldens(render_results, output_dir, goldens): - for result in render_results: - if result['result'] != RESULT_OK: - continue - - out_tif_basename = f"{result['name']}.tif" - out_tif_name = f'{output_dir}/{out_tif_basename}' - golden_path = goldens.get(out_tif_basename) - if not golden_path: - result['result'] = RESULT_FAILED_NO_GOLDEN - result['result_code'] = 1 - elif not same_image(golden_path, out_tif_name): - result['result'] = RESULT_FAILED_IMAGE_DIFF - result['result_code'] = 1 - - return render_results - if __name__ == "__main__": parser = ArgParseImpl() parser.add_argument('--test', help='Configuration of the test', required=True) @@ -120,43 +90,26 @@ if __name__ == "__main__": parser.add_argument('--output_dir', help='Output Directory', required=True) parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)') parser.add_argument('--vk_icd', help='Path to VK ICD file') - parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against') args, _ = parser.parse_known_args(sys.argv[1:]) - test = parse_test_config_from_path(args.test) + test = test_config.parse_from_path(args.test) output_dir, results = \ - run_test(args.gltf_viewer, - test, - args.output_dir, - opengl_lib=args.opengl_lib, - vk_icd=args.vk_icd) + _render_test_config(args.gltf_viewer, + test, + args.output_dir, + opengl_lib=args.opengl_lib, + vk_icd=args.vk_icd) - do_compare = False - # The presence of this argument indicates comparison against a set of goldens. - if args.golden_branch: - # prepare goldens working directory - tmp_golden_dir = '/tmp/renderdiff-goldens' - mkdir_p(tmp_golden_dir) - - # Download the golden repo into the current working directory - golden_manager = GoldenManager(os.getcwd()) - golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch) - - goldens = { - os.path.basename(fpath) : fpath for fpath in \ - glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True) - } - results = compare_goldens(results, output_dir, goldens) - do_compare = True - - with open(f'{output_dir}/results.json', 'w') as f: + with open(f'{output_dir}/render_results.json', 'w') as f: f.write(json.dumps(results)) shutil.copy2(args.test, f'{output_dir}/test.json') failed = [f" {k['name']}" for k in results if k['result'] != RESULT_OK] success_count = len(results) - len(failed) - op = 'tested' if do_compare else 'rendered' - important_print(f'Successfully {op} {success_count} / {len(results)}' + - ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) + important_print(f'Successfully rendered {success_count} / {len(results)} tests' + + ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) + + if len(failed) > 0: + exit(1) diff --git a/test/renderdiff/src/results.py b/test/renderdiff/src/results.py new file mode 100644 index 0000000000..6b75edeca6 --- /dev/null +++ b/test/renderdiff/src/results.py @@ -0,0 +1,3 @@ +RESULT_OK = 'ok' +RESULT_FAILED = 'failed' +RESULT_MISSING = 'missing' diff --git a/test/renderdiff/src/parse_test_json.py b/test/renderdiff/src/test_config.py similarity index 98% rename from test/renderdiff/src/parse_test_json.py rename to test/renderdiff/src/test_config.py index 36e7e6521f..c7b5a024f5 100644 --- a/test/renderdiff/src/parse_test_json.py +++ b/test/renderdiff/src/test_config.py @@ -129,7 +129,7 @@ def _remove_comments_from_json_txt(json_txt): res.append(line) return '\n'.join(res) -def parse_test_config_from_path(config_path): +def parse_from_path(config_path): with open(config_path, 'r') as f: json_txt = json.loads(_remove_comments_from_json_txt(f.read())) return RenderTestConfig(json_txt) diff --git a/test/renderdiff/src/update_golden.py b/test/renderdiff/src/update_golden.py index 58c0cd028a..e534c42ae6 100644 --- a/test/renderdiff/src/update_golden.py +++ b/test/renderdiff/src/update_golden.py @@ -90,7 +90,7 @@ def _interactive_mode(base_golden_dir): if prompt_helper( f'Generate the new goldens from your local ' \ f'Filament branch? (branch={cur_branch})') == PROMPT_YES: - code, res = execute('bash ./test/renderdiff/test.sh generate', + code, res = execute('bash ./test/renderdiff/generate.sh', capture_output=False) if code != 0: print('Failed to generate new goldens') diff --git a/test/renderdiff/src/utils.py b/test/renderdiff/src/utils.py index 9dc7bb442d..6922493268 100644 --- a/test/renderdiff/src/utils.py +++ b/test/renderdiff/src/utils.py @@ -106,3 +106,13 @@ def mkdir_p(path_str): def mv_f(src_str, dst_str): src = pathlib.Path(src_str) src.replace(dst_str) + +def important_print(msg): + lines = msg.split('\n') + max_len = max([len(l) for l in lines]) + print('-' * (max_len + 8)) + for line in lines: + diff = max_len - len(line) + information = f'--- {line} ' + (' ' * diff) + '---' + print(information) + print('-' * (max_len + 8)) diff --git a/test/renderdiff/test.sh b/test/renderdiff/test.sh index 726b3e9965..7109fd91a6 100755 --- a/test/renderdiff/test.sh +++ b/test/renderdiff/test.sh @@ -14,71 +14,15 @@ #!/usr/bin/bash -OUTPUT_DIR="$(pwd)/out/renderdiff_tests" -RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff" -BUILD_COMMON_DIR="$(pwd)/build/common" -MESA_DIR="$(pwd)/mesa/out/" -VENV_DIR="$(pwd)/venv" +source `dirname $0`/src/preamble.sh -os_name=$(uname -s) -if [[ "$os_name" == "Linux" ]]; then - MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu" -elif [[ "$os_name" == "Darwin" ]]; then - MESA_LIB_DIR="${MESA_DIR}lib" -else - echo "Unsupported platform for renderdiff tests" - exit 1 -fi - -function start_() { - if [[ "$GITHUB_WORKFLOW" ]]; then - set -ex - else - if [ ! -d ${MESA_LIB_DIR} ]; then - bash ${BUILD_COMMON_DIR}/get-mesa.sh - fi - - # Install python deps - python3 -m venv ${VENV_DIR} - source ${VENV_DIR}/bin/activate - - NEEDED_PYTHON_DEPS=("numpy" "tifffile") - for cmd in "${NEEDED_PYTHON_DEPS[@]}"; do - if ! python3 -m pip show -q "${cmd}"; then - python3 -m pip install ${cmd} - fi - done - fi - -} - -function end_() { - if [[ "$GITHUB_WORKFLOW" ]]; then - set +ex - else - deactivate # End python virtual env - fi -} - -# Following steps are taken: -# - Get and build mesa -# - Build gltf_viewer -# - Run the python script that runs the test -# - Zip up the result - -GOLDEN_BRANCH_PARAM='--golden_branch=main' - -if [ "$1" == "generate" ]; then - GOLDEN_BRANCH_PARAM='' -fi +GOLDEN_DIR=$(pwd)/golden_images start_ && \ - mkdir -p ${OUTPUT_DIR} && \ - CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer && \ - python3 ${RENDERDIFF_TEST_DIR}/src/run.py \ - --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ - --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ - --output_dir=${OUTPUT_DIR} \ - --opengl_lib=${MESA_LIB_DIR} \ - ${GOLDEN_BRANCH_PARAM} && \ + bash `dirname $0`/generate.sh && \ + mkdir -p ${GOLDEN_DIR} && \ + python3 ${RENDERDIFF_TEST_DIR}/src/golden_manager.py --output=${GOLDEN_DIR} && \ + python3 ${RENDERDIFF_TEST_DIR}/src/compare.py \ + --src=${GOLDEN_DIR} \ + --dest=${OUTPUT_DIR} && \ end_