renderdiff: script for updating golden images (#8709)

Adding a python script to enable updating new goldens into
a staging branch in the golden repo (filament-assets).

The same script can be used in github workflow to automatically
create a golden staging branch. This will be useful for users
without access to a mac (the only platform for generating
goldens as of now).

test/renderdiff/src/golden_manager.py

@@ -1,14 +1,42 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import shutil
+import re
 
-from utils import execute, ArgParseImpl
+from utils import execute, ArgParseImpl, mkdir_p
 
 GOLDENS_DIR = 'renderdiff'
 
+ACCESS_TYPE_TOKEN = 'token'
+ACCESS_TYPE_SSH = 'ssh'
+ACCESS_TYPE_READ_ONLY = 'read-only'
+
+def _read_git_config(curdir):
+  with open(os.path.join(curdir, './.git/config'), 'r') as f:
+    return f.read()
+
+def _write_git_config(curdir, config_str):
+  with open(os.path.join(curdir, './.git/config'), 'w') as f:
+    return f.write(config_str)
+
 class GoldenManager:
-  def __init__(self, working_dir, access_token=None):
+  def __init__(self, working_dir, access_type=ACCESS_TYPE_READ_ONLY, access_token=None):
     self.working_dir_ = working_dir
     self.access_token_ = access_token
+    self.access_type_ = access_type
     assert os.path.isdir(self.working_dir_),\
         f"working directory {self.working_dir_} does not exist"
     self._prepare()
@@ -16,16 +44,36 @@ class GoldenManager:
   def _assets_dir(self):
     return os.path.join(self.working_dir_, "filament-assets")
 
+  # Returns the directory containing the goldens
+  def directory(self):
+    return os.path.join(self._assets_dir(), GOLDENS_DIR)
+
+  def _get_repo_url(self):
+    protocol = ''
+    protocol_separator = ''
+    if self.access_type_ == ACCESS_TYPE_SSH:
+      protocol = 'git@'
+      protocol_separator = ':'
+    else:
+      protocol = 'https://' + \
+        (f'x-access-token:{self.access_token_}@' if self.access_token_ else '')
+      protocol_separator = '/'
+    return f'{protocol}github.com{protocol_separator}google/filament-assets.git'
+
   def _prepare(self):
     assets_dir = self._assets_dir()
     if not os.path.exists(assets_dir):
-      access_token_part = ''
-      if self.access_token_:
-        access_token_part = f'x-access-token:{self.access_token_}@'
       execute(
-          f'git clone --depth=1 https://{access_token_part}github.com/google/filament-assets.git',
-          cwd=self.working_dir_)
+          f'git clone --depth=1 {self._get_repo_url()}',
+          cwd=self.working_dir_,
+          capture_output=False
+      )
     else:
+      if self.access_type_ == ACCESS_TYPE_SSH:
+        config = _read_git_config(self._assets_dir())
+        https_url = r'https://github\.com\/google\/filament\.git'
+        config = re.sub(https_url, self._get_repo_url(), config)
+        _write_git_config(self._assets_dir(), config)
       self.update()
 
   def update(self):
@@ -41,31 +89,46 @@ class GoldenManager:
     assets_dir = self._assets_dir()
     self._git_exec(f'checkout main')
     self._git_exec(f'merge --no-ff {branch}')
-    if push_to_remote and self.access_token_:
+    if push_to_remote and \
+       (self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH):
       self._git_exec(f'push origin main')
+      self.update()
 
-  def source_from_and_commit(self, src_dir, commit_msg, branch, push_to_remote=False):
+  def source_from(self, src_dir, commit_msg, branch,
+                  updates=[], deletes=[], push_to_remote=False):
     assets_dir = self._assets_dir()
     self._git_exec(f'checkout main')
     # Force create the branch (note will overwrite the old branch)
     self._git_exec(f'switch -C {branch}')
     rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
-    execute(f'rm -rf {rdiff_dir}')
-    execute(f'mkdir -p {rdiff_dir}')
-    shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True)
-    self._git_exec(f'add {GOLDENS_DIR}')
+    if len(updates) == 0 and len(deletes) == 0:
+      shutil.rmtree(rdiff_dir, ignore_errors=True)
+      mkdir_p(rdiff_dir)
+      shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True)
+      self._git_exec(f'add {GOLDENS_DIR}')
+    else:
+      for f in deletes:
+        self._git_exec(f'remove {os.path.join(GOLDENS_DIR, f)}')
+      for f in updates:
+        shutil.copy2(
+          os.path.join(src_dir, f),
+          os.path.join(rdiff_dir, f))
+        self._git_exec(f'add {os.path.join(GOLDENS_DIR, f)}')
 
     TMP_GOLDEN_COMMIT_FILE = '/tmp/golden_commit.txt'
 
     with open(TMP_GOLDEN_COMMIT_FILE, 'w') as f:
       f.write(commit_msg)
-    self._git_exec(f'commit -F {TMP_GOLDEN_COMMIT_FILE}')
-    if push_to_remote and self.access_token_:
-      self._git_exec(f'push -f origin ${branch}')
+    self._git_exec(f'commit -a -F {TMP_GOLDEN_COMMIT_FILE}')
+    if push_to_remote and \
+       (self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH):
+      self._git_exec(f'push -f origin {branch}')
+      self.update()
 
   def download_to(self, dest_dir, branch='main'):
+    self._git_exec(f'checkout {branch}')
     assets_dir = self._assets_dir()
-    execute(f'mkdir -p {dest_dir}')
+    mkdir_p(dest_dir)
     rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
     shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True)
 

test/renderdiff/src/image_diff.py

@@ -1,3 +1,17 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import tifffile
 import numpy
 

test/renderdiff/src/run.py

@@ -16,8 +16,9 @@ import sys
 import os
 import json
 import glob
+import shutil
 
-from utils import execute, ArgParseImpl
+from utils import execute, ArgParseImpl, mkdir_p, mv_f
 from parse_test_json import parse_test_config_from_path
 from golden_manager import GoldenManager
 from image_diff import same_image
@@ -46,7 +47,7 @@ def run_test(gltf_viewer,
   assert os.access(gltf_viewer, os.X_OK)
 
   named_output_dir = os.path.join(output_dir, test_config.name)
-  execute(f'mkdir -p {named_output_dir}')
+  mkdir_p(named_output_dir)
 
   results = []
   for test in test_config.tests:
@@ -82,9 +83,8 @@ def run_test(gltf_viewer,
           result = RESULT_OK
           out_tif_basename = f'{out_name}.tif'
           out_tif_name = f'{named_output_dir}/{out_tif_basename}'
-          execute(f'mv -f {test.name}0.tif {out_tif_name}', capture_output=False)
-          execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json',
-                  capture_output=False)
+          mv_f(f'{test.name}0.tif', out_tif_name)
+          mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json')
         else:
           result = RESULT_FAILED_TO_RENDER
           important_print(f'{test_desc} rendering failed with error={out_code}')
@@ -132,11 +132,12 @@ if __name__ == "__main__":
              opengl_lib=args.opengl_lib,
              vk_icd=args.vk_icd)
 
+  do_compare = False
   # The presence of this argument indicates comparison against a set of goldens.
   if args.golden_branch:
     # prepare goldens working directory
     tmp_golden_dir = '/tmp/renderdiff-goldens'
-    execute(f'mkdir -p {tmp_golden_dir}')
+    mkdir_p(tmp_golden_dir)
 
     # Download the golden repo into the current working directory
     golden_manager = GoldenManager(os.getcwd())
@@ -147,13 +148,15 @@ if __name__ == "__main__":
       glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
     }
     results = compare_goldens(results, output_dir, goldens)
-
+    do_compare = True
 
   with open(f'{output_dir}/results.json', 'w') as f:
     f.write(json.dumps(results))
-  execute(f'cp {args.test} {output_dir}/test.json')
+
+  shutil.copy2(args.test, f'{output_dir}/test.json')
 
   failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
   success_count = len(results) - len(failed)
-  important_print(f'Successfully tested {success_count} / {len(results)}' +
+  op = 'tested' if do_compare else 'rendered'
+  important_print(f'Successfully {op} {success_count} / {len(results)}' +
                   ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/update_golden.py

@@ -0,0 +1,171 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import glob
+import time
+
+from golden_manager import GoldenManager, ACCESS_TYPE_SSH, ACCESS_TYPE_TOKEN
+from image_diff import same_image
+
+from utils import execute, ArgParseImpl
+from utils import prompt_helper, PROMPT_YES, PROMPT_NO
+
+def line_prompt(prompt, validator=lambda a:True):
+  while True:
+    res = input(f'{prompt} => ').strip()
+    if validator(res):
+      return res
+  return None
+
+CONFIG_NEW_SRC_DIR = 'goldens_dir'
+CONFIG_GOLDENS_BRANCH = 'goldens_branch'
+CONFIG_GOLDENS_UPDATES = 'goldens_updates'
+CONFIG_GOLDENS_DELETES = 'goldens_deletes'
+CONFIG_AUTO_COMMIT = 'auto-commit'
+CONFIG_COMMIT_MSG = 'commit_msg'
+
+def _get_current_branch():
+  code, res = execute('git branch --show-current')
+  return res.strip()
+
+def _file_as_str(fpath):
+  with open(fpath, 'r') as f:
+    return f.read()
+
+def _do_update(golden_manager, config):
+  deletes = config[CONFIG_GOLDENS_DELETES]
+  updates = config[CONFIG_GOLDENS_UPDATES]
+  if len(deletes) == 0 and len(updates) == 0:
+    print('Nothing to update. Exiting...')
+    exit(0)
+
+  branch = config[CONFIG_GOLDENS_BRANCH]
+  src_dir = config[CONFIG_NEW_SRC_DIR]
+  auto_commit = config[CONFIG_AUTO_COMMIT]
+  commit_msg = config[CONFIG_COMMIT_MSG]
+  golden_manager.source_from(src_dir, commit_msg, branch,
+                             updates=updates,
+                             deletes=deletes,
+                             push_to_remote=auto_commit)
+
+def _get_deletes_updates(update_dir, golden_dir):
+  ret_delete = []
+  ret_update = []
+  for ext in ['tif', 'json']:
+    base = set(glob.glob(f'./**/*.{ext}', root_dir=golden_dir, recursive=True))
+    new = set(glob.glob(f'./**/*.{ext}', root_dir=update_dir, recursive=True))
+
+    delete = list(base - new)
+    update = list(new - base)
+
+    for fpath in base.intersection(new):
+      base_fpath = os.path.join(golden_dir, fpath)
+      new_fpath = os.path.join(update_dir, fpath)
+      if (ext == 'tif' and not same_image(new_fpath, base_fpath)) or \
+         (ext == 'json' and _file_as_str(new_fpath) != _file_as_str(base_fpath)):
+        update.append(fpath)
+
+    ret_update += update
+    ret_delete += delete
+
+  return ret_delete, ret_update
+
+# Ask a bunch of questions to gather the configuration for the update
+def _interactive_mode(base_golden_dir):
+  config = {}
+  cur_branch = _get_current_branch()
+  if prompt_helper(
+          f'Generate the new goldens from your local ' \
+          f'Filament branch? (branch={cur_branch})') == PROMPT_YES:
+    code, res = execute('bash ./test/renderdiff/test.sh generate',
+            capture_output=False)
+    if code != 0:
+      print('Failed to generate new goldens')
+      exit(1)
+    config[CONFIG_NEW_SRC_DIR] = os.path.join(os.getcwd(), './out/renderdiff_tests/')
+  else:
+    def validator(src_dir):
+      if not os.path.exists(src_dir):
+        print(f'Cannot find directory {src_dir}. Please try again.')
+        return False
+      return True
+
+    config[CONFIG_NEW_SRC_DIR] = line_prompt(
+        'Please provide path of directory containing new goldens',
+        validator)
+
+  if prompt_helper(f'Update new goldens to branch={cur_branch}? '
+                   '(Note that this refers to a branch in the goldens repo, not the Filament repo.)'
+                   ) == PROMPT_YES:
+    config[CONFIG_GOLDENS_BRANCH] = cur_branch
+  else:
+    config[CONFIG_GOLDENS_BRANCH] = line_prompt('Please provide new branch name for update')
+
+  if prompt_helper(f'Provide a commit message?') == PROMPT_YES:
+    config[CONFIG_COMMIT_MSG] = line_prompt('Message:')
+  else:
+    config[CONFIG_COMMIT_MSG] = f'Update {time.time()} from filament ({cur_branch})'
+
+  new_golden_dir = config[CONFIG_NEW_SRC_DIR]
+  deletes, updates = _get_deletes_updates(new_golden_dir, base_golden_dir)
+  if len(deletes) + len(updates) != 0:
+    prompt = 'The following files will be changed:\n' + \
+        '\n'.join([f'  {fname} [delete]' for fname in deletes]) + \
+        '\n'.join([f'  {fname} [update]' for fname in updates]) + \
+        '\nIs that ok?'
+    if prompt_helper(prompt) == PROMPT_YES:
+      config[CONFIG_GOLDENS_DELETES] = deletes
+      config[CONFIG_GOLDENS_UPDATES] = updates
+    else:
+      # We cannot proceed if user answered no.
+      exit(1)
+  else:
+    config[CONFIG_GOLDENS_DELETES] = []
+    config[CONFIG_GOLDENS_UPDATES] = []
+
+  config[CONFIG_AUTO_COMMIT] = \
+      prompt_helper(f'Commit golden repo changes to remote?') == PROMPT_YES
+  return config
+
+if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--branch', help='Branch of the golden repo to write to')
+  parser.add_argument('--source', help='Directory containing the new goldens')
+  parser.add_argument('--commit-msg', help='Message for the commit to the golden repo')
+  parser.add_argument('--golden-repo-token', help='Access token for the golden repo')
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+  config = {}
+  golden_manager = GoldenManager(
+      os.getcwd(),
+      access_type=ACCESS_TYPE_SSH if not args.golden_repo_token else ACCESS_TYPE_TOKEN,
+      access_token=args.golden_repo_token
+  )
+  base_golden_dir = golden_manager.directory()
+  if args.branch and args.source and args.commit_msg:
+    assert os.path.exists(args.source), f'{args.source} (--source) directory not found'
+    deletes, updates = _get_deletes_updates(args.source, base_golden_dir)
+    config = {
+        CONFIG_AUTO_COMMIT: True,
+        CONFIG_GOLDENS_BRANCH: args.branch,
+        CONFIG_NEW_SRC_DIR: args.source,
+        CONFIG_GOLDENS_UPDATES: updates,
+        CONFIG_GOLDENS_DELETES: deletes,
+        CONFIG_COMMIT_MSG: args.commit_msg,
+    }
+  else:
+    config = _interactive_mode(base_golden_dir)
+  _do_update(golden_manager, config)

test/renderdiff/src/utils.py

@@ -16,6 +16,7 @@ import subprocess
 import os
 import argparse
 import sys
+import pathlib
 
 def execute(cmd,
             cwd=None,
@@ -66,3 +67,42 @@ class ArgParseImpl(argparse.ArgumentParser):
     sys.stderr.write('error: %s\n' % message)
     self.print_help()
     sys.exit(1)
+
+
+PROMPT_YES = 'y'
+PROMPT_NO = 'n'
+PROMPT_YES_NO = f'{PROMPT_YES}{PROMPT_NO}'
+
+class GetCh:
+  def __init__(self):
+    pass
+
+  def __call__(self):
+    import sys, tty, termios
+    fd = sys.stdin.fileno()
+    old_settings = termios.tcgetattr(fd)
+    try:
+      tty.setraw(sys.stdin.fileno())
+      ch = sys.stdin.read(1)
+    finally:
+      termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+    return ch
+
+getch = GetCh()
+
+def prompt_helper(prompt_str, keys=PROMPT_YES_NO):
+  while True:
+    print(f'{prompt_str}: [' + ', '.join(keys) + '] => ', end='', flush=True)
+    val = getch()
+    print(val)
+    if val in keys or ord(val) == 3: # If user pressed Ctrl+c
+      if ord(val) == 3:
+        exit(1)
+    return val
+
+def mkdir_p(path_str):
+  pathlib.Path(path_str).mkdir(parents=True, exist_ok=True)
+
+def mv_f(src_str, dst_str):
+  src = pathlib.Path(src_str)
+  src.replace(dst_str)

test/renderdiff/src/workflow_presubmit_msg.py

@@ -17,19 +17,29 @@ from utils import execute
 def get_last_commit():
   res, o = execute('git log -1')
   commit, author, date, _, title, *desc = o.split('\n')
+  commit = commit.split(' ')[1]
+  title = title.strip()
 
   desc = [l.strip() for l in desc[1:]]
-  if len(desc) > 0 and len(desc[0]) == 0:
+  while len(desc) > 0 and len(desc[0]) == 0:
     desc = desc[1:]
 
   return (
-    commit.split(' ')[1],
-    title.strip(),
+    commit,
+    title,
     desc
   )
 
 def sanitized_split(line, split_atom='\n'):
-  return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom))))
+  return list(
+    filter(
+      lambda x: len(x) > 0,
+      map(
+        lambda x: x.strip(),
+        line.split(split_atom)
+      )
+    )
+  )
 
 RDIFF_UPDATE_GOLDEN_STR = 'RDIFF_UPDATE_GOLDEN'