diff --git a/test/renderdiff/src/golden_manager.py b/test/renderdiff/src/golden_manager.py index 6daabc1fda..7db4162d6e 100644 --- a/test/renderdiff/src/golden_manager.py +++ b/test/renderdiff/src/golden_manager.py @@ -1,14 +1,42 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import shutil +import re -from utils import execute, ArgParseImpl +from utils import execute, ArgParseImpl, mkdir_p GOLDENS_DIR = 'renderdiff' +ACCESS_TYPE_TOKEN = 'token' +ACCESS_TYPE_SSH = 'ssh' +ACCESS_TYPE_READ_ONLY = 'read-only' + +def _read_git_config(curdir): + with open(os.path.join(curdir, './.git/config'), 'r') as f: + return f.read() + +def _write_git_config(curdir, config_str): + with open(os.path.join(curdir, './.git/config'), 'w') as f: + return f.write(config_str) + class GoldenManager: - def __init__(self, working_dir, access_token=None): + def __init__(self, working_dir, access_type=ACCESS_TYPE_READ_ONLY, access_token=None): self.working_dir_ = working_dir self.access_token_ = access_token + self.access_type_ = access_type assert os.path.isdir(self.working_dir_),\ f"working directory {self.working_dir_} does not exist" self._prepare() @@ -16,16 +44,36 @@ class GoldenManager: def _assets_dir(self): return os.path.join(self.working_dir_, "filament-assets") + # Returns the directory containing the goldens + def directory(self): + return os.path.join(self._assets_dir(), GOLDENS_DIR) + + def _get_repo_url(self): + protocol = '' + protocol_separator = '' + if self.access_type_ == ACCESS_TYPE_SSH: + protocol = 'git@' + protocol_separator = ':' + else: + protocol = 'https://' + \ + (f'x-access-token:{self.access_token_}@' if self.access_token_ else '') + protocol_separator = '/' + return f'{protocol}github.com{protocol_separator}google/filament-assets.git' + def _prepare(self): assets_dir = self._assets_dir() if not os.path.exists(assets_dir): - access_token_part = '' - if self.access_token_: - access_token_part = f'x-access-token:{self.access_token_}@' execute( - f'git clone --depth=1 https://{access_token_part}github.com/google/filament-assets.git', - cwd=self.working_dir_) + f'git clone --depth=1 {self._get_repo_url()}', + cwd=self.working_dir_, + capture_output=False + ) else: + if self.access_type_ == ACCESS_TYPE_SSH: + config = _read_git_config(self._assets_dir()) + https_url = r'https://github\.com\/google\/filament\.git' + config = re.sub(https_url, self._get_repo_url(), config) + _write_git_config(self._assets_dir(), config) self.update() def update(self): @@ -41,31 +89,46 @@ class GoldenManager: assets_dir = self._assets_dir() self._git_exec(f'checkout main') self._git_exec(f'merge --no-ff {branch}') - if push_to_remote and self.access_token_: + if push_to_remote and \ + (self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH): self._git_exec(f'push origin main') + self.update() - def source_from_and_commit(self, src_dir, commit_msg, branch, push_to_remote=False): + def source_from(self, src_dir, commit_msg, branch, + updates=[], deletes=[], push_to_remote=False): assets_dir = self._assets_dir() self._git_exec(f'checkout main') # Force create the branch (note will overwrite the old branch) self._git_exec(f'switch -C {branch}') rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR) - execute(f'rm -rf {rdiff_dir}') - execute(f'mkdir -p {rdiff_dir}') - shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True) - self._git_exec(f'add {GOLDENS_DIR}') + if len(updates) == 0 and len(deletes) == 0: + shutil.rmtree(rdiff_dir, ignore_errors=True) + mkdir_p(rdiff_dir) + shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True) + self._git_exec(f'add {GOLDENS_DIR}') + else: + for f in deletes: + self._git_exec(f'remove {os.path.join(GOLDENS_DIR, f)}') + for f in updates: + shutil.copy2( + os.path.join(src_dir, f), + os.path.join(rdiff_dir, f)) + self._git_exec(f'add {os.path.join(GOLDENS_DIR, f)}') TMP_GOLDEN_COMMIT_FILE = '/tmp/golden_commit.txt' with open(TMP_GOLDEN_COMMIT_FILE, 'w') as f: f.write(commit_msg) - self._git_exec(f'commit -F {TMP_GOLDEN_COMMIT_FILE}') - if push_to_remote and self.access_token_: - self._git_exec(f'push -f origin ${branch}') + self._git_exec(f'commit -a -F {TMP_GOLDEN_COMMIT_FILE}') + if push_to_remote and \ + (self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH): + self._git_exec(f'push -f origin {branch}') + self.update() def download_to(self, dest_dir, branch='main'): + self._git_exec(f'checkout {branch}') assets_dir = self._assets_dir() - execute(f'mkdir -p {dest_dir}') + mkdir_p(dest_dir) rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR) shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True) diff --git a/test/renderdiff/src/image_diff.py b/test/renderdiff/src/image_diff.py index b4c3d3eedb..23b83a0f8e 100644 --- a/test/renderdiff/src/image_diff.py +++ b/test/renderdiff/src/image_diff.py @@ -1,3 +1,17 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import tifffile import numpy diff --git a/test/renderdiff/src/run.py b/test/renderdiff/src/run.py index 63334faa03..524c0a2f94 100644 --- a/test/renderdiff/src/run.py +++ b/test/renderdiff/src/run.py @@ -16,8 +16,9 @@ import sys import os import json import glob +import shutil -from utils import execute, ArgParseImpl +from utils import execute, ArgParseImpl, mkdir_p, mv_f from parse_test_json import parse_test_config_from_path from golden_manager import GoldenManager from image_diff import same_image @@ -46,7 +47,7 @@ def run_test(gltf_viewer, assert os.access(gltf_viewer, os.X_OK) named_output_dir = os.path.join(output_dir, test_config.name) - execute(f'mkdir -p {named_output_dir}') + mkdir_p(named_output_dir) results = [] for test in test_config.tests: @@ -82,9 +83,8 @@ def run_test(gltf_viewer, result = RESULT_OK out_tif_basename = f'{out_name}.tif' out_tif_name = f'{named_output_dir}/{out_tif_basename}' - execute(f'mv -f {test.name}0.tif {out_tif_name}', capture_output=False) - execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', - capture_output=False) + mv_f(f'{test.name}0.tif', out_tif_name) + mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json') else: result = RESULT_FAILED_TO_RENDER important_print(f'{test_desc} rendering failed with error={out_code}') @@ -132,11 +132,12 @@ if __name__ == "__main__": opengl_lib=args.opengl_lib, vk_icd=args.vk_icd) + do_compare = False # The presence of this argument indicates comparison against a set of goldens. if args.golden_branch: # prepare goldens working directory tmp_golden_dir = '/tmp/renderdiff-goldens' - execute(f'mkdir -p {tmp_golden_dir}') + mkdir_p(tmp_golden_dir) # Download the golden repo into the current working directory golden_manager = GoldenManager(os.getcwd()) @@ -147,13 +148,15 @@ if __name__ == "__main__": glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True) } results = compare_goldens(results, output_dir, goldens) - + do_compare = True with open(f'{output_dir}/results.json', 'w') as f: f.write(json.dumps(results)) - execute(f'cp {args.test} {output_dir}/test.json') + + shutil.copy2(args.test, f'{output_dir}/test.json') failed = [f" {k['name']}" for k in results if k['result'] != RESULT_OK] success_count = len(results) - len(failed) - important_print(f'Successfully tested {success_count} / {len(results)}' + + op = 'tested' if do_compare else 'rendered' + important_print(f'Successfully {op} {success_count} / {len(results)}' + ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) diff --git a/test/renderdiff/src/update_golden.py b/test/renderdiff/src/update_golden.py new file mode 100644 index 0000000000..58c0cd028a --- /dev/null +++ b/test/renderdiff/src/update_golden.py @@ -0,0 +1,171 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import glob +import time + +from golden_manager import GoldenManager, ACCESS_TYPE_SSH, ACCESS_TYPE_TOKEN +from image_diff import same_image + +from utils import execute, ArgParseImpl +from utils import prompt_helper, PROMPT_YES, PROMPT_NO + +def line_prompt(prompt, validator=lambda a:True): + while True: + res = input(f'{prompt} => ').strip() + if validator(res): + return res + return None + +CONFIG_NEW_SRC_DIR = 'goldens_dir' +CONFIG_GOLDENS_BRANCH = 'goldens_branch' +CONFIG_GOLDENS_UPDATES = 'goldens_updates' +CONFIG_GOLDENS_DELETES = 'goldens_deletes' +CONFIG_AUTO_COMMIT = 'auto-commit' +CONFIG_COMMIT_MSG = 'commit_msg' + +def _get_current_branch(): + code, res = execute('git branch --show-current') + return res.strip() + +def _file_as_str(fpath): + with open(fpath, 'r') as f: + return f.read() + +def _do_update(golden_manager, config): + deletes = config[CONFIG_GOLDENS_DELETES] + updates = config[CONFIG_GOLDENS_UPDATES] + if len(deletes) == 0 and len(updates) == 0: + print('Nothing to update. Exiting...') + exit(0) + + branch = config[CONFIG_GOLDENS_BRANCH] + src_dir = config[CONFIG_NEW_SRC_DIR] + auto_commit = config[CONFIG_AUTO_COMMIT] + commit_msg = config[CONFIG_COMMIT_MSG] + golden_manager.source_from(src_dir, commit_msg, branch, + updates=updates, + deletes=deletes, + push_to_remote=auto_commit) + +def _get_deletes_updates(update_dir, golden_dir): + ret_delete = [] + ret_update = [] + for ext in ['tif', 'json']: + base = set(glob.glob(f'./**/*.{ext}', root_dir=golden_dir, recursive=True)) + new = set(glob.glob(f'./**/*.{ext}', root_dir=update_dir, recursive=True)) + + delete = list(base - new) + update = list(new - base) + + for fpath in base.intersection(new): + base_fpath = os.path.join(golden_dir, fpath) + new_fpath = os.path.join(update_dir, fpath) + if (ext == 'tif' and not same_image(new_fpath, base_fpath)) or \ + (ext == 'json' and _file_as_str(new_fpath) != _file_as_str(base_fpath)): + update.append(fpath) + + ret_update += update + ret_delete += delete + + return ret_delete, ret_update + +# Ask a bunch of questions to gather the configuration for the update +def _interactive_mode(base_golden_dir): + config = {} + cur_branch = _get_current_branch() + if prompt_helper( + f'Generate the new goldens from your local ' \ + f'Filament branch? (branch={cur_branch})') == PROMPT_YES: + code, res = execute('bash ./test/renderdiff/test.sh generate', + capture_output=False) + if code != 0: + print('Failed to generate new goldens') + exit(1) + config[CONFIG_NEW_SRC_DIR] = os.path.join(os.getcwd(), './out/renderdiff_tests/') + else: + def validator(src_dir): + if not os.path.exists(src_dir): + print(f'Cannot find directory {src_dir}. Please try again.') + return False + return True + + config[CONFIG_NEW_SRC_DIR] = line_prompt( + 'Please provide path of directory containing new goldens', + validator) + + if prompt_helper(f'Update new goldens to branch={cur_branch}? ' + '(Note that this refers to a branch in the goldens repo, not the Filament repo.)' + ) == PROMPT_YES: + config[CONFIG_GOLDENS_BRANCH] = cur_branch + else: + config[CONFIG_GOLDENS_BRANCH] = line_prompt('Please provide new branch name for update') + + if prompt_helper(f'Provide a commit message?') == PROMPT_YES: + config[CONFIG_COMMIT_MSG] = line_prompt('Message:') + else: + config[CONFIG_COMMIT_MSG] = f'Update {time.time()} from filament ({cur_branch})' + + new_golden_dir = config[CONFIG_NEW_SRC_DIR] + deletes, updates = _get_deletes_updates(new_golden_dir, base_golden_dir) + if len(deletes) + len(updates) != 0: + prompt = 'The following files will be changed:\n' + \ + '\n'.join([f' {fname} [delete]' for fname in deletes]) + \ + '\n'.join([f' {fname} [update]' for fname in updates]) + \ + '\nIs that ok?' + if prompt_helper(prompt) == PROMPT_YES: + config[CONFIG_GOLDENS_DELETES] = deletes + config[CONFIG_GOLDENS_UPDATES] = updates + else: + # We cannot proceed if user answered no. + exit(1) + else: + config[CONFIG_GOLDENS_DELETES] = [] + config[CONFIG_GOLDENS_UPDATES] = [] + + config[CONFIG_AUTO_COMMIT] = \ + prompt_helper(f'Commit golden repo changes to remote?') == PROMPT_YES + return config + +if __name__ == "__main__": + parser = ArgParseImpl() + parser.add_argument('--branch', help='Branch of the golden repo to write to') + parser.add_argument('--source', help='Directory containing the new goldens') + parser.add_argument('--commit-msg', help='Message for the commit to the golden repo') + parser.add_argument('--golden-repo-token', help='Access token for the golden repo') + + args, _ = parser.parse_known_args(sys.argv[1:]) + config = {} + golden_manager = GoldenManager( + os.getcwd(), + access_type=ACCESS_TYPE_SSH if not args.golden_repo_token else ACCESS_TYPE_TOKEN, + access_token=args.golden_repo_token + ) + base_golden_dir = golden_manager.directory() + if args.branch and args.source and args.commit_msg: + assert os.path.exists(args.source), f'{args.source} (--source) directory not found' + deletes, updates = _get_deletes_updates(args.source, base_golden_dir) + config = { + CONFIG_AUTO_COMMIT: True, + CONFIG_GOLDENS_BRANCH: args.branch, + CONFIG_NEW_SRC_DIR: args.source, + CONFIG_GOLDENS_UPDATES: updates, + CONFIG_GOLDENS_DELETES: deletes, + CONFIG_COMMIT_MSG: args.commit_msg, + } + else: + config = _interactive_mode(base_golden_dir) + _do_update(golden_manager, config) diff --git a/test/renderdiff/src/utils.py b/test/renderdiff/src/utils.py index f708d7a659..9dc7bb442d 100644 --- a/test/renderdiff/src/utils.py +++ b/test/renderdiff/src/utils.py @@ -16,6 +16,7 @@ import subprocess import os import argparse import sys +import pathlib def execute(cmd, cwd=None, @@ -66,3 +67,42 @@ class ArgParseImpl(argparse.ArgumentParser): sys.stderr.write('error: %s\n' % message) self.print_help() sys.exit(1) + + +PROMPT_YES = 'y' +PROMPT_NO = 'n' +PROMPT_YES_NO = f'{PROMPT_YES}{PROMPT_NO}' + +class GetCh: + def __init__(self): + pass + + def __call__(self): + import sys, tty, termios + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + +getch = GetCh() + +def prompt_helper(prompt_str, keys=PROMPT_YES_NO): + while True: + print(f'{prompt_str}: [' + ', '.join(keys) + '] => ', end='', flush=True) + val = getch() + print(val) + if val in keys or ord(val) == 3: # If user pressed Ctrl+c + if ord(val) == 3: + exit(1) + return val + +def mkdir_p(path_str): + pathlib.Path(path_str).mkdir(parents=True, exist_ok=True) + +def mv_f(src_str, dst_str): + src = pathlib.Path(src_str) + src.replace(dst_str) diff --git a/test/renderdiff/src/workflow_presubmit_msg.py b/test/renderdiff/src/workflow_presubmit_msg.py index e971aa7b33..dfcd57deb9 100644 --- a/test/renderdiff/src/workflow_presubmit_msg.py +++ b/test/renderdiff/src/workflow_presubmit_msg.py @@ -17,19 +17,29 @@ from utils import execute def get_last_commit(): res, o = execute('git log -1') commit, author, date, _, title, *desc = o.split('\n') + commit = commit.split(' ')[1] + title = title.strip() desc = [l.strip() for l in desc[1:]] - if len(desc) > 0 and len(desc[0]) == 0: + while len(desc) > 0 and len(desc[0]) == 0: desc = desc[1:] return ( - commit.split(' ')[1], - title.strip(), + commit, + title, desc ) def sanitized_split(line, split_atom='\n'): - return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom)))) + return list( + filter( + lambda x: len(x) > 0, + map( + lambda x: x.strip(), + line.split(split_atom) + ) + ) + ) RDIFF_UPDATE_GOLDEN_STR = 'RDIFF_UPDATE_GOLDEN' diff --git a/test/renderdiff/test.sh b/test/renderdiff/test.sh index d219419a58..6a53bc039c 100755 --- a/test/renderdiff/test.sh +++ b/test/renderdiff/test.sh @@ -18,6 +18,7 @@ OUTPUT_DIR="$(pwd)/out/renderdiff_tests" RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff" TEST_UTILS_DIR="$(pwd)/test/utils" MESA_DIR="$(pwd)/mesa/out/" +VENV_DIR="$(pwd)/venv" os_name=$(uname -s) if [[ "$os_name" == "Linux" ]]; then @@ -29,9 +30,33 @@ else exit 1 fi -function prepare_mesa() { - if [ ! -d ${MESA_LIB_DIR} ]; then - bash ${TEST_UTILS_DIR}/get_mesa.sh +function start_() { + if [[ "$GITHUB_WORKFLOW" ]]; then + set -ex + else + if [ ! -d ${MESA_LIB_DIR} ]; then + bash ${TEST_UTILS_DIR}/get_mesa.sh + fi + + # Install python deps + python3 -m venv ${VENV_DIR} + source ${VENV_DIR}/bin/activate + + NEEDED_PYTHON_DEPS=("numpy" "tifffile") + for cmd in "${NEEDED_PYTHON_DEPS[@]}"; do + if ! python3 -m pip show -q "${cmd}"; then + python3 -m pip install ${cmd} + fi + done + fi + +} + +function end_() { + if [[ "$GITHUB_WORKFLOW" ]]; then + set +ex + else + deactivate # End python virtual env fi } @@ -41,12 +66,19 @@ function prepare_mesa() { # - Run the python script that runs the test # - Zip up the result -set -ex && prepare_mesa && \ +GOLDEN_BRANCH_PARAM='--golden_branch=main' + +if [ "$1" == "generate" ]; then + GOLDEN_BRANCH_PARAM='' +fi + +start_ && \ mkdir -p ${OUTPUT_DIR} && \ - CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer && \ - python3 ${RENDERDIFF_TEST_DIR}/src/run.py \ + CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer && \ + python3 ${RENDERDIFF_TEST_DIR}/src/run.py \ --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ --output_dir=${OUTPUT_DIR} \ --opengl_lib=${MESA_LIB_DIR} \ - --golden_branch=main + ${GOLDEN_BRANCH_PARAM} && \ + end_ diff --git a/test/utils/get_mesa.sh b/test/utils/get_mesa.sh index 508d5a5eb7..89809626d4 100755 --- a/test/utils/get_mesa.sh +++ b/test/utils/get_mesa.sh @@ -14,9 +14,9 @@ #!/usr/bin/bash -set -x +set -e if [[ "$GITHUB_WORKFLOW" ]]; then - set -e + set -x fi OS_NAME=$(uname -s) @@ -35,7 +35,7 @@ source ${ORIG_DIR}/venv/bin/activate NEEDED_PYTHON_DEPS=("mako" "setuptools" "pyyaml") for cmd in "${NEEDED_PYTHON_DEPS[@]}"; do - if ! python3 -m pip show "${cmd}" >/dev/null 2>&1; then + if ! python3 -m pip show -q "${cmd}" >/dev/null 2>&1; then python3 -m pip install ${cmd} fi done @@ -145,6 +145,6 @@ deactivate popd if [[ "$GITHUB_WORKFLOW" ]]; then - set +e + set +x fi -set +x +set +e