renderdiff: add golden repo support (#8689)

- Add GoldenManager to manage access to the repo containing the
  goldens
- Add tif comparison code
- Enable comparison by default for actual test

test/renderdiff/src/golden_manager.py

@@ -0,0 +1,80 @@
+import os
+import shutil
+
+from utils import execute, ArgParseImpl
+
+GOLDENS_DIR = 'renderdiff'
+
+class GoldenManager:
+  def __init__(self, working_dir, access_token=None):
+    self.working_dir_ = working_dir
+    self.access_token_ = access_token
+    assert os.path.isdir(self.working_dir_),\
+        f"working directory {self.working_dir_} does not exist"
+    self._prepare()
+
+  def _assets_dir(self):
+    return os.path.join(self.working_dir_, "filament-assets")
+
+  def _prepare(self):
+    assets_dir = self._assets_dir()
+    if not os.path.exists(assets_dir):
+      access_token_part = ''
+      if self.access_token_:
+        access_token_part = f'x-access-token:{self.access_token_}@'
+      execute(
+          f'git clone --depth=1 https://{access_token_part}github.com/google/filament-assets.git',
+          cwd=self.working_dir_)
+    else:
+      self.update()
+
+  def update(self):
+    self._git_exec('fetch')
+    self._git_exec('checkout main')
+    self._git_exec('rebase')
+
+  def _git_exec(self, cmd):
+    execute(f'git {cmd}', cwd=self._assets_dir(), capture_output=False)
+
+  def merge_to_main(self, branch, push_to_remote=False):
+    self.update()
+    assets_dir = self._assets_dir()
+    self._git_exec(f'checkout main')
+    self._git_exec(f'merge --no-ff {branch}')
+    if push_to_remote and self.access_token_:
+      self._git_exec(f'push origin main')
+
+  def source_from_and_commit(self, src_dir, commit_msg, branch, push_to_remote=False):
+    assets_dir = self._assets_dir()
+    self._git_exec(f'checkout main')
+    # Force create the branch (note will overwrite the old branch)
+    self._git_exec(f'switch -C {branch}')
+    rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
+    execute(f'rm -rf {rdiff_dir}')
+    execute(f'mkdir -p {rdiff_dir}')
+    shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True)
+    self._git_exec(f'add {GOLDENS_DIR}')
+
+    TMP_GOLDEN_COMMIT_FILE = '/tmp/golden_commit.txt'
+
+    with open(TMP_GOLDEN_COMMIT_FILE, 'w') as f:
+      f.write(commit_msg)
+    self._git_exec(f'commit -F {TMP_GOLDEN_COMMIT_FILE}')
+    if push_to_remote and self.access_token_:
+      self._git_exec(f'push -f origin ${branch}')
+
+  def download_to(self, dest_dir, branch='main'):
+    assets_dir = self._assets_dir()
+    execute(f'mkdir -p {dest_dir}')
+    rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
+    shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True)
+
+# For testing only
+if __name__ == "__main__":
+  golden_manager = GoldenManager(os.getcwd())
+  # golden_manager.source_from_and_commit(
+  #     os.path.join(os.getcwd(), 'out/renderdiff_tests'),
+  #     'First commit (local)',
+  #     branch='branch-test')
+  # golden_manager.merge_to_main('branch-test', push_to_remote=True)
+  # golden_manager.download_to(os.path.join(os.getcwd(), 'tmp/goldens'))

test/renderdiff/src/image_diff.py

@@ -0,0 +1,29 @@
+import tifffile
+import numpy
+
+def same_image(tiff_file_a, tiff_file_b):
+  try:
+    img1_data = tifffile.imread(tiff_file_a)
+    img2_data = tifffile.imread(tiff_file_b)
+
+    # If the dimensions (height, width, number of channels, number of pages/frames)
+    # are different, the images are not the same.
+    if img1_data.shape != img2_data.shape:
+      print(f"Images have different shapes: {img1_data.shape} vs {img2_data.shape}")
+      return False
+
+    # numpy.array_equal() checks if two arrays have the same shape and elements.
+    if numpy.array_equal(img1_data, img2_data):
+      return True
+    else:
+      return False
+
+  except FileNotFoundError:
+    print(f"Error: One or both files not found ('{file_path1}', '{file_path2}').")
+    return False
+  except tifffile.TiffFileError as e:
+    print(f"Error: One or both files are not valid TIFF files or could not be read. Details: {e}")
+    return False
+  except Exception as e:
+    print(f"An unexpected error occurred: {e}")
+    return False

test/renderdiff/src/run.py

@@ -14,9 +14,13 @@
 
 import sys
 import os
+import json
+import glob
 
 from utils import execute, ArgParseImpl
 from parse_test_json import parse_test_config_from_path
+from golden_manager import GoldenManager
+from image_diff import same_image
 
 def important_print(msg):
   lines = msg.split('\n')
@@ -28,8 +32,16 @@ def important_print(msg):
     print(information)
   print('-' * (max_len + 8))
 
-def render_test(gltf_viewer, test_config, output_dir,
-                opengl_lib=None, vk_icd=None):
+RESULT_OK = 'ok'
+RESULT_FAILED_TO_RENDER = 'failed-to-render'
+RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff'
+RESULT_FAILED_NO_GOLDEN = 'failed-no-golden'
+
+def run_test(gltf_viewer,
+             test_config,
+             output_dir,
+             opengl_lib=None,
+             vk_icd=None):
   assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
   assert os.access(gltf_viewer, os.X_OK)
 
@@ -60,43 +72,46 @@ def render_test(gltf_viewer, test_config, output_dir,
 
         important_print(f'Rendering {test_desc}')
 
-        res, _ = execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
-                env=env, capture_output=False)
+        out_code, _ = execute(
+          f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
+          env=env, capture_output=False
+        )
 
-        if res == 0:
-          execute(f'mv -f {test.name}0.tif {named_output_dir}/{out_name}.tif', capture_output=False)
-          execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', capture_output=False)
+        result = ''
+        if out_code == 0:
+          result = RESULT_OK
+          out_tif_basename = f'{out_name}.tif'
+          out_tif_name = f'{named_output_dir}/{out_tif_basename}'
+          execute(f'mv -f {test.name}0.tif {out_tif_name}', capture_output=False)
+          execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json',
+                  capture_output=False)
         else:
-          important_print(f'{test_desc} failed with error={res}')
-        print('')
+          result = RESULT_FAILED_TO_RENDER
+          important_print(f'{test_desc} rendering failed with error={out_code}')
 
-        results.append((out_name, res))
-  return results
+        results.append({
+          'name': out_name,
+          'result': result,
+          'result_code': out_code,
+        })
+  return named_output_dir, results
 
-GOLDENS_DIR = 'renderdiff_goldens'
+def compare_goldens(render_results, output_dir, goldens):
+  for result in render_results:
+    if result['result'] != RESULT_OK:
+      continue
 
-# We pull the goldens from the filament-assets repo
-def pull_goldens(output_dir):
-  assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
-  golden_dir = os.path.join(output_dir, "golden")
-  assets_dir = os.path.join(output_dir, "filament-assets")
+    out_tif_basename = f"{result['name']}.tif"
+    out_tif_name = f'{output_dir}/{out_tif_basename}'
+    golden_path = goldens.get(out_tif_basename)
+    if not golden_path:
+      result['result'] = RESULT_FAILED_NO_GOLDEN
+      result['result_code'] = 1
+    elif not same_image(golden_path, out_tif_name):
+      result['result'] = RESULT_FAILED_IMAGE_DIFF
+      result['result_code'] = 1
 
-  if not os.path.exists(assets_dir):
-    execute('git clone --depth 1 git@github.com:google/filament-assets.git', cwd=output_dir)
-  else:
-    execute('git fetch', cwd=assets_dir)
-    execute('git checkout main ', cwd=assets_dir)
-    execute('git rebase', cwd=assets_dir)
-
-  if os.path.exists(golden_dir):
-    execute('rm -f goldens/*', cwd=output_dir)
-  execute(f'cp filament-assets/{GOLDENS_DIR}/* goldens', cwd=output_dir)
-
-def push_goldens(output_dir, test_name, filter_func=lambda a:True):
-  for test in test_config.tests:
-    for backend in test_config.backends:
-      for model in test.models:
-        pass
+  return render_results
 
 if __name__ == "__main__":
   parser = ArgParseImpl()
@@ -105,12 +120,40 @@ if __name__ == "__main__":
   parser.add_argument('--output_dir', help='Output Directory', required=True)
   parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
   parser.add_argument('--vk_icd', help='Path to VK ICD file')
+  parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against')
 
   args, _ = parser.parse_known_args(sys.argv[1:])
   test = parse_test_config_from_path(args.test)
-  render_result = render_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd)
 
-  failed = [f'   {tname}' for tname, res in render_result if res != 0]
-  success_count = len(render_result) - len(failed  )
-  important_print(f'Successfully rendered {success_count} / {len(render_result)}' +
+  output_dir, results = \
+    run_test(args.gltf_viewer,
+             test,
+             args.output_dir,
+             opengl_lib=args.opengl_lib,
+             vk_icd=args.vk_icd)
+
+  # The presence of this argument indicates comparison against a set of goldens.
+  if args.golden_branch:
+    # prepare goldens working directory
+    tmp_golden_dir = '/tmp/renderdiff-goldens'
+    execute(f'mkdir -p {tmp_golden_dir}')
+
+    # Download the golden repo into the current working directory
+    golden_manager = GoldenManager(os.getcwd())
+    golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch)
+
+    goldens = {
+      os.path.basename(fpath) : fpath for fpath in \
+      glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
+    }
+    results = compare_goldens(results, output_dir, goldens)
+
+
+  with open(f'{output_dir}/results.json', 'w') as f:
+    f.write(json.dumps(results))
+  execute(f'cp {args.test} {output_dir}/test.json')
+
+  failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
+  success_count = len(results) - len(failed)
+  important_print(f'Successfully tested {success_count} / {len(results)}' +
                   ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/workflow_presubmit_msg.py

@@ -25,7 +25,8 @@ def get_last_commit():
   return (
     commit.split(' ')[1],
     title.strip(),
-    desc)
+    desc
+  )
 
 def sanitized_split(line, split_atom='\n'):
   return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom))))

test/renderdiff/test.sh

@@ -43,9 +43,10 @@ function prepare_mesa() {
 
 set -ex && prepare_mesa && \
     mkdir -p ${OUTPUT_DIR} && \
-        CXX=`which clang++` CC=`which clang` ./build.sh -X ${MESA_DIR} -p desktop debug gltf_viewer && \
+        CXX=`which clang++` CC=`which clang` ./build.sh -f -X ${MESA_DIR} -p desktop debug gltf_viewer && \
         python3 ${RENDERDIFF_TEST_DIR}/src/run.py \
             --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \
             --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \
             --output_dir=${OUTPUT_DIR} \
-            --opengl_lib=${MESA_LIB_DIR}
+            --opengl_lib=${MESA_LIB_DIR} \
+            --golden_branch=main