From 28ecf5c35d2323dd7db762fe55641d4bd6d0df10 Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Wed, 7 May 2025 14:20:22 -0700 Subject: [PATCH] renderdiff: add golden repo support (#8689) - Add GoldenManager to manage access to the repo containing the goldens - Add tif comparison code - Enable comparison by default for actual test --- .github/workflows/presubmit.yml | 12 +- test/renderdiff/src/golden_manager.py | 80 ++++++++++++ test/renderdiff/src/image_diff.py | 29 +++++ test/renderdiff/src/run.py | 117 ++++++++++++------ test/renderdiff/src/workflow_presubmit_msg.py | 3 +- test/renderdiff/test.sh | 5 +- 6 files changed, 201 insertions(+), 45 deletions(-) create mode 100644 test/renderdiff/src/golden_manager.py create mode 100644 test/renderdiff/src/image_diff.py diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 42d2af00a9..f9b738f154 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -124,13 +124,15 @@ jobs: - uses: ./.github/actions/mac-prereq - name: Cache Mesa and deps id: mesa-cache - uses: actions/cache@v4 # Use a specific version + uses: actions/cache@v4 with: path: mesa key: ${{ runner.os }}-mesa-deps-2-${{ vars.MESA_VERSION }} - - name: Get Mesa - id: mesa-prereq - run: bash test/utils/get_mesa.sh + - name: Prerequisites + id: prereqs + run: | + bash test/utils/get_mesa.sh + pip install tifffile numpy - name: Run Test run: bash test/renderdiff/test.sh - uses: actions/upload-artifact@v4 @@ -151,7 +153,7 @@ jobs: - name: Run test run: ./out/cmake-debug/libs/filamat/test_filamat --gtest_filter=MaterialCompiler.Wgsl* - code-correcteness: + code-correctness: name: code-correctness runs-on: 'macos-14-xlarge' steps: diff --git a/test/renderdiff/src/golden_manager.py b/test/renderdiff/src/golden_manager.py new file mode 100644 index 0000000000..6daabc1fda --- /dev/null +++ b/test/renderdiff/src/golden_manager.py @@ -0,0 +1,80 @@ +import os +import shutil + +from utils import execute, ArgParseImpl + +GOLDENS_DIR = 'renderdiff' + +class GoldenManager: + def __init__(self, working_dir, access_token=None): + self.working_dir_ = working_dir + self.access_token_ = access_token + assert os.path.isdir(self.working_dir_),\ + f"working directory {self.working_dir_} does not exist" + self._prepare() + + def _assets_dir(self): + return os.path.join(self.working_dir_, "filament-assets") + + def _prepare(self): + assets_dir = self._assets_dir() + if not os.path.exists(assets_dir): + access_token_part = '' + if self.access_token_: + access_token_part = f'x-access-token:{self.access_token_}@' + execute( + f'git clone --depth=1 https://{access_token_part}github.com/google/filament-assets.git', + cwd=self.working_dir_) + else: + self.update() + + def update(self): + self._git_exec('fetch') + self._git_exec('checkout main') + self._git_exec('rebase') + + def _git_exec(self, cmd): + execute(f'git {cmd}', cwd=self._assets_dir(), capture_output=False) + + def merge_to_main(self, branch, push_to_remote=False): + self.update() + assets_dir = self._assets_dir() + self._git_exec(f'checkout main') + self._git_exec(f'merge --no-ff {branch}') + if push_to_remote and self.access_token_: + self._git_exec(f'push origin main') + + def source_from_and_commit(self, src_dir, commit_msg, branch, push_to_remote=False): + assets_dir = self._assets_dir() + self._git_exec(f'checkout main') + # Force create the branch (note will overwrite the old branch) + self._git_exec(f'switch -C {branch}') + rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR) + execute(f'rm -rf {rdiff_dir}') + execute(f'mkdir -p {rdiff_dir}') + shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True) + self._git_exec(f'add {GOLDENS_DIR}') + + TMP_GOLDEN_COMMIT_FILE = '/tmp/golden_commit.txt' + + with open(TMP_GOLDEN_COMMIT_FILE, 'w') as f: + f.write(commit_msg) + self._git_exec(f'commit -F {TMP_GOLDEN_COMMIT_FILE}') + if push_to_remote and self.access_token_: + self._git_exec(f'push -f origin ${branch}') + + def download_to(self, dest_dir, branch='main'): + assets_dir = self._assets_dir() + execute(f'mkdir -p {dest_dir}') + rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR) + shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True) + +# For testing only +if __name__ == "__main__": + golden_manager = GoldenManager(os.getcwd()) + # golden_manager.source_from_and_commit( + # os.path.join(os.getcwd(), 'out/renderdiff_tests'), + # 'First commit (local)', + # branch='branch-test') + # golden_manager.merge_to_main('branch-test', push_to_remote=True) + # golden_manager.download_to(os.path.join(os.getcwd(), 'tmp/goldens')) diff --git a/test/renderdiff/src/image_diff.py b/test/renderdiff/src/image_diff.py new file mode 100644 index 0000000000..b4c3d3eedb --- /dev/null +++ b/test/renderdiff/src/image_diff.py @@ -0,0 +1,29 @@ +import tifffile +import numpy + +def same_image(tiff_file_a, tiff_file_b): + try: + img1_data = tifffile.imread(tiff_file_a) + img2_data = tifffile.imread(tiff_file_b) + + # If the dimensions (height, width, number of channels, number of pages/frames) + # are different, the images are not the same. + if img1_data.shape != img2_data.shape: + print(f"Images have different shapes: {img1_data.shape} vs {img2_data.shape}") + return False + + # numpy.array_equal() checks if two arrays have the same shape and elements. + if numpy.array_equal(img1_data, img2_data): + return True + else: + return False + + except FileNotFoundError: + print(f"Error: One or both files not found ('{file_path1}', '{file_path2}').") + return False + except tifffile.TiffFileError as e: + print(f"Error: One or both files are not valid TIFF files or could not be read. Details: {e}") + return False + except Exception as e: + print(f"An unexpected error occurred: {e}") + return False diff --git a/test/renderdiff/src/run.py b/test/renderdiff/src/run.py index 15c5663eba..63334faa03 100644 --- a/test/renderdiff/src/run.py +++ b/test/renderdiff/src/run.py @@ -14,9 +14,13 @@ import sys import os +import json +import glob from utils import execute, ArgParseImpl from parse_test_json import parse_test_config_from_path +from golden_manager import GoldenManager +from image_diff import same_image def important_print(msg): lines = msg.split('\n') @@ -28,8 +32,16 @@ def important_print(msg): print(information) print('-' * (max_len + 8)) -def render_test(gltf_viewer, test_config, output_dir, - opengl_lib=None, vk_icd=None): +RESULT_OK = 'ok' +RESULT_FAILED_TO_RENDER = 'failed-to-render' +RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff' +RESULT_FAILED_NO_GOLDEN = 'failed-no-golden' + +def run_test(gltf_viewer, + test_config, + output_dir, + opengl_lib=None, + vk_icd=None): assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist" assert os.access(gltf_viewer, os.X_OK) @@ -60,43 +72,46 @@ def render_test(gltf_viewer, test_config, output_dir, important_print(f'Rendering {test_desc}') - res, _ = execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless', - env=env, capture_output=False) + out_code, _ = execute( + f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless', + env=env, capture_output=False + ) - if res == 0: - execute(f'mv -f {test.name}0.tif {named_output_dir}/{out_name}.tif', capture_output=False) - execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', capture_output=False) + result = '' + if out_code == 0: + result = RESULT_OK + out_tif_basename = f'{out_name}.tif' + out_tif_name = f'{named_output_dir}/{out_tif_basename}' + execute(f'mv -f {test.name}0.tif {out_tif_name}', capture_output=False) + execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', + capture_output=False) else: - important_print(f'{test_desc} failed with error={res}') - print('') + result = RESULT_FAILED_TO_RENDER + important_print(f'{test_desc} rendering failed with error={out_code}') - results.append((out_name, res)) - return results + results.append({ + 'name': out_name, + 'result': result, + 'result_code': out_code, + }) + return named_output_dir, results -GOLDENS_DIR = 'renderdiff_goldens' +def compare_goldens(render_results, output_dir, goldens): + for result in render_results: + if result['result'] != RESULT_OK: + continue -# We pull the goldens from the filament-assets repo -def pull_goldens(output_dir): - assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist" - golden_dir = os.path.join(output_dir, "golden") - assets_dir = os.path.join(output_dir, "filament-assets") + out_tif_basename = f"{result['name']}.tif" + out_tif_name = f'{output_dir}/{out_tif_basename}' + golden_path = goldens.get(out_tif_basename) + if not golden_path: + result['result'] = RESULT_FAILED_NO_GOLDEN + result['result_code'] = 1 + elif not same_image(golden_path, out_tif_name): + result['result'] = RESULT_FAILED_IMAGE_DIFF + result['result_code'] = 1 - if not os.path.exists(assets_dir): - execute('git clone --depth 1 git@github.com:google/filament-assets.git', cwd=output_dir) - else: - execute('git fetch', cwd=assets_dir) - execute('git checkout main ', cwd=assets_dir) - execute('git rebase', cwd=assets_dir) - - if os.path.exists(golden_dir): - execute('rm -f goldens/*', cwd=output_dir) - execute(f'cp filament-assets/{GOLDENS_DIR}/* goldens', cwd=output_dir) - -def push_goldens(output_dir, test_name, filter_func=lambda a:True): - for test in test_config.tests: - for backend in test_config.backends: - for model in test.models: - pass + return render_results if __name__ == "__main__": parser = ArgParseImpl() @@ -105,12 +120,40 @@ if __name__ == "__main__": parser.add_argument('--output_dir', help='Output Directory', required=True) parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)') parser.add_argument('--vk_icd', help='Path to VK ICD file') + parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against') args, _ = parser.parse_known_args(sys.argv[1:]) test = parse_test_config_from_path(args.test) - render_result = render_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd) - failed = [f' {tname}' for tname, res in render_result if res != 0] - success_count = len(render_result) - len(failed ) - important_print(f'Successfully rendered {success_count} / {len(render_result)}' + + output_dir, results = \ + run_test(args.gltf_viewer, + test, + args.output_dir, + opengl_lib=args.opengl_lib, + vk_icd=args.vk_icd) + + # The presence of this argument indicates comparison against a set of goldens. + if args.golden_branch: + # prepare goldens working directory + tmp_golden_dir = '/tmp/renderdiff-goldens' + execute(f'mkdir -p {tmp_golden_dir}') + + # Download the golden repo into the current working directory + golden_manager = GoldenManager(os.getcwd()) + golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch) + + goldens = { + os.path.basename(fpath) : fpath for fpath in \ + glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True) + } + results = compare_goldens(results, output_dir, goldens) + + + with open(f'{output_dir}/results.json', 'w') as f: + f.write(json.dumps(results)) + execute(f'cp {args.test} {output_dir}/test.json') + + failed = [f" {k['name']}" for k in results if k['result'] != RESULT_OK] + success_count = len(results) - len(failed) + important_print(f'Successfully tested {success_count} / {len(results)}' + ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) diff --git a/test/renderdiff/src/workflow_presubmit_msg.py b/test/renderdiff/src/workflow_presubmit_msg.py index 06d8975364..e971aa7b33 100644 --- a/test/renderdiff/src/workflow_presubmit_msg.py +++ b/test/renderdiff/src/workflow_presubmit_msg.py @@ -25,7 +25,8 @@ def get_last_commit(): return ( commit.split(' ')[1], title.strip(), - desc) + desc + ) def sanitized_split(line, split_atom='\n'): return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom)))) diff --git a/test/renderdiff/test.sh b/test/renderdiff/test.sh index 354cdf6236..d219419a58 100755 --- a/test/renderdiff/test.sh +++ b/test/renderdiff/test.sh @@ -43,9 +43,10 @@ function prepare_mesa() { set -ex && prepare_mesa && \ mkdir -p ${OUTPUT_DIR} && \ - CXX=`which clang++` CC=`which clang` ./build.sh -X ${MESA_DIR} -p desktop debug gltf_viewer && \ + CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer && \ python3 ${RENDERDIFF_TEST_DIR}/src/run.py \ --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ --output_dir=${OUTPUT_DIR} \ - --opengl_lib=${MESA_LIB_DIR} + --opengl_lib=${MESA_LIB_DIR} \ + --golden_branch=main