renderdiff: separate comparison from rendering (#8733)

- Move the comparison logic into its own script
- Add entry point bash script to generate the renderings
- Separate the preamble bash logic into its own file

test/renderdiff/src/compare.py

@@ -0,0 +1,54 @@
+import glob
+import os
+import sys
+import pprint
+import json
+
+from utils import execute, ArgParseImpl, important_print
+from image_diff import same_image
+from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING
+
+def _compare_goldens(base_dir, comparison_dir):
+  render_results = {}
+  base_files = glob.glob(os.path.join(base_dir, "./**/*.tif"))
+  for golden_file in base_files:
+    base_fname = os.path.abspath(golden_file)
+    test_case = base_fname.replace(f'{os.path.abspath(base_dir)}/', '')
+    comp_fname = os.path.abspath(os.path.join(comparison_dir, test_case))
+    if not os.path.exists(comp_fname):
+      print(f'file name not found: {comp_fname}')
+      render_results[test_case] = RESULT_MISSING
+      continue
+    if not same_image(base_fname, comp_fname):
+      render_results[test_case] = RESULT_FAILED
+    else:
+      render_results[test_case] = RESULT_OK
+  return render_results
+
+if __name__ == '__main__':
+  parser = ArgParseImpl()
+  parser.add_argument('--src', help='Directory of the base of the diff.', required=True)
+  parser.add_argument('--dest', help='Directory of the comparison of the diff.')
+  parser.add_argument('--out', help='Directory of output for the result of the diff.')
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+
+  dest = args.dest
+  if not dest:
+    print('Assume the default renderdiff output folder')
+    dest = os.path.join(os.getcwd(), './out/renderdiff_tests')
+  assert os.path.exists(dest), f"Destination folder={dest} does not exist."
+
+  results = _compare_goldens(args.src, dest)
+
+  if args.out:
+    assert os.path.exists(arg.out), f"Output folder={dest} does not exist."
+    with open(os.path.join(args.out, "compare_results.json", 'w')) as f:
+      f.write(json.dumps(results))
+
+  failed = [f"   {k}" for k in results.keys() if results[k] != RESULT_OK]
+  success_count = len(results) - len(failed)
+  important_print(f'Successfully compared {success_count} / {len(results)} images' +
+                    ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
+  if len(failed) > 0:
+    exit(1)

test/renderdiff/src/golden_manager.py

@@ -15,6 +15,7 @@
 import os
 import shutil
 import re
+import sys
 
 from utils import execute, ArgParseImpl, mkdir_p
 
@@ -132,12 +133,19 @@ class GoldenManager:
     rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
     shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True)
 
-# For testing only
+# The main entry point will enable download content of a branch to a directory
 if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--branch', type=str, help='Branch of the golden repo', default='main')
+  parser.add_argument('--output', type=str, help='Directory to download to', required=True)
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+
+  # prepare goldens working directory
+  golden_dir = args.output
+  assert os.path.isdir(golden_dir),\
+    f"Output directory {golden_dir} does not exist"
+
+  # Download the golden repo into the current working directory
   golden_manager = GoldenManager(os.getcwd())
-  # golden_manager.source_from_and_commit(
-  #     os.path.join(os.getcwd(), 'out/renderdiff_tests'),
-  #     'First commit (local)',
-  #     branch='branch-test')
-  # golden_manager.merge_to_main('branch-test', push_to_remote=True)
-  # golden_manager.download_to(os.path.join(os.getcwd(), 'tmp/goldens'))
+  golden_manager.download_to(golden_dir, branch=args.branch)

test/renderdiff/src/preamble.sh

@@ -0,0 +1,45 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/bash
+
+# Sets up the environment for scripts in test/renderdiff/
+
+OUTPUT_DIR="$(pwd)/out/renderdiff_tests"
+RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff"
+MESA_DIR="$(pwd)/mesa/out/"
+VENV_DIR="$(pwd)/venv"
+BUILD_COMMON_DIR="$(pwd)/build/common"
+
+os_name=$(uname -s)
+if [[ "$os_name" == "Linux" ]]; then
+    MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu"
+elif [[ "$os_name" == "Darwin" ]]; then
+    MESA_LIB_DIR="${MESA_DIR}lib"
+else
+    echo "Unsupported platform for renderdiff tests"
+    exit 1
+fi
+
+function start_() {
+    if [[ "$GITHUB_WORKFLOW" ]]; then
+        set -ex
+    fi
+}
+
+function end_() {
+    if [[ "$GITHUB_WORKFLOW" ]]; then
+        set +ex
+    fi
+}

test/renderdiff/src/render.py

@@ -18,31 +18,18 @@ import json
 import glob
 import shutil
 
-from utils import execute, ArgParseImpl, mkdir_p, mv_f
-from parse_test_json import parse_test_config_from_path
+from utils import execute, ArgParseImpl, mkdir_p, mv_f, important_print
+
+import test_config
 from golden_manager import GoldenManager
 from image_diff import same_image
+from results import RESULT_OK, RESULT_FAILED
 
-def important_print(msg):
-  lines = msg.split('\n')
-  max_len = max([len(l) for l in lines])
-  print('-' * (max_len + 8))
-  for line in lines:
-    diff = max_len - len(line)
-    information = f'--- {line} ' + (' ' * diff) + '---'
-    print(information)
-  print('-' * (max_len + 8))
-
-RESULT_OK = 'ok'
-RESULT_FAILED_TO_RENDER = 'failed-to-render'
-RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff'
-RESULT_FAILED_NO_GOLDEN = 'failed-no-golden'
-
-def run_test(gltf_viewer,
-             test_config,
-             output_dir,
-             opengl_lib=None,
-             vk_icd=None):
+def _render_test_config(gltf_viewer,
+                        test_config,
+                        output_dir,
+                        opengl_lib=None,
+                        vk_icd=None):
   assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
   assert os.access(gltf_viewer, os.X_OK)
 
@@ -86,7 +73,7 @@ def run_test(gltf_viewer,
           mv_f(f'{test.name}0.tif', out_tif_name)
           mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json')
         else:
-          result = RESULT_FAILED_TO_RENDER
+          result = RESULT_FAILED
           important_print(f'{test_desc} rendering failed with error={out_code}')
 
         results.append({
@@ -96,23 +83,6 @@ def run_test(gltf_viewer,
         })
   return named_output_dir, results
 
-def compare_goldens(render_results, output_dir, goldens):
-  for result in render_results:
-    if result['result'] != RESULT_OK:
-      continue
-
-    out_tif_basename = f"{result['name']}.tif"
-    out_tif_name = f'{output_dir}/{out_tif_basename}'
-    golden_path = goldens.get(out_tif_basename)
-    if not golden_path:
-      result['result'] = RESULT_FAILED_NO_GOLDEN
-      result['result_code'] = 1
-    elif not same_image(golden_path, out_tif_name):
-      result['result'] = RESULT_FAILED_IMAGE_DIFF
-      result['result_code'] = 1
-
-  return render_results
-
 if __name__ == "__main__":
   parser = ArgParseImpl()
   parser.add_argument('--test', help='Configuration of the test', required=True)
@@ -120,43 +90,26 @@ if __name__ == "__main__":
   parser.add_argument('--output_dir', help='Output Directory', required=True)
   parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
   parser.add_argument('--vk_icd', help='Path to VK ICD file')
-  parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against')
 
   args, _ = parser.parse_known_args(sys.argv[1:])
-  test = parse_test_config_from_path(args.test)
+  test = test_config.parse_from_path(args.test)
 
   output_dir, results = \
-    run_test(args.gltf_viewer,
-             test,
-             args.output_dir,
-             opengl_lib=args.opengl_lib,
-             vk_icd=args.vk_icd)
+    _render_test_config(args.gltf_viewer,
+                        test,
+                        args.output_dir,
+                        opengl_lib=args.opengl_lib,
+                        vk_icd=args.vk_icd)
 
-  do_compare = False
-  # The presence of this argument indicates comparison against a set of goldens.
-  if args.golden_branch:
-    # prepare goldens working directory
-    tmp_golden_dir = '/tmp/renderdiff-goldens'
-    mkdir_p(tmp_golden_dir)
-
-    # Download the golden repo into the current working directory
-    golden_manager = GoldenManager(os.getcwd())
-    golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch)
-
-    goldens = {
-      os.path.basename(fpath) : fpath for fpath in \
-      glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
-    }
-    results = compare_goldens(results, output_dir, goldens)
-    do_compare = True
-
-  with open(f'{output_dir}/results.json', 'w') as f:
+  with open(f'{output_dir}/render_results.json', 'w') as f:
     f.write(json.dumps(results))
 
   shutil.copy2(args.test, f'{output_dir}/test.json')
 
   failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
   success_count = len(results) - len(failed)
-  op = 'tested' if do_compare else 'rendered'
-  important_print(f'Successfully {op} {success_count} / {len(results)}' +
-                  ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
+  important_print(f'Successfully rendered {success_count} / {len(results)} tests' +
+                   ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
+
+  if len(failed) > 0:
+    exit(1)

test/renderdiff/src/results.py

@@ -0,0 +1,3 @@
+RESULT_OK = 'ok'
+RESULT_FAILED = 'failed'
+RESULT_MISSING = 'missing'

test/renderdiff/src/test_config.py

@@ -129,7 +129,7 @@ def _remove_comments_from_json_txt(json_txt):
     res.append(line)
   return '\n'.join(res)
 
-def parse_test_config_from_path(config_path):
+def parse_from_path(config_path):
   with open(config_path, 'r') as f:
     json_txt = json.loads(_remove_comments_from_json_txt(f.read()))
     return RenderTestConfig(json_txt)

test/renderdiff/src/update_golden.py

@@ -90,7 +90,7 @@ def _interactive_mode(base_golden_dir):
   if prompt_helper(
           f'Generate the new goldens from your local ' \
           f'Filament branch? (branch={cur_branch})') == PROMPT_YES:
-    code, res = execute('bash ./test/renderdiff/test.sh generate',
+    code, res = execute('bash ./test/renderdiff/generate.sh',
             capture_output=False)
     if code != 0:
       print('Failed to generate new goldens')

test/renderdiff/src/utils.py

@@ -106,3 +106,13 @@ def mkdir_p(path_str):
 def mv_f(src_str, dst_str):
   src = pathlib.Path(src_str)
   src.replace(dst_str)
+
+def important_print(msg):
+  lines = msg.split('\n')
+  max_len = max([len(l) for l in lines])
+  print('-' * (max_len + 8))
+  for line in lines:
+    diff = max_len - len(line)
+    information = f'--- {line} ' + (' ' * diff) + '---'
+    print(information)
+  print('-' * (max_len + 8))