renderdiff: add initial code for updating goldens (#8349)

- Code for parsing tag in commit message
- Add placeholder for pulling and updating goldens in the script.

test/renderdiff/src/parse_test_json.py

@@ -0,0 +1,143 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from utils import execute, ArgParseImpl
+
+import glob
+from itertools import chain
+import json
+import sys
+import os
+from os import path
+
+def _is_list_of_strings(field):
+  return isinstance(field, list) and\
+      all(isinstance(item, str) for item in field)
+
+def _is_string(s):
+    return isinstance(s, str)
+
+def _is_dict(s):
+    return isinstance(s, dict)
+
+class RenderingConfig():
+  def __init__(self, data):
+    assert 'name' in data
+    assert _is_string(data['name'])
+    self.name = data['name']
+
+    assert 'rendering' in data
+    assert _is_dict(data['rendering'])
+    self.rendering = data['rendering']
+
+class PresetConfig(RenderingConfig):
+  def __init__(self, data, existing_models):
+    RenderingConfig.__init__(self, data)
+    models = data.get('models')
+    if models:
+      assert _is_list_of_strings(models)
+      assert all(m in existing_models for m in models)
+      self.models = models
+
+class TestConfig(RenderingConfig):
+  def __init__(self, data, existing_models, presets):
+    RenderingConfig.__init__(self, data)
+    description = data.get('description')
+    if description:
+      assert _is_string(description)
+      self.description = description
+
+    apply_presets = data.get('apply_presets')
+    rendering = {}
+    preset_models = []
+    if apply_presets:
+      given_presets = {p.name: p for p in presets}
+      assert all((name in given_presets) for name in apply_presets),\
+        f'used preset {name} which is not in {given_presets}'
+      for preset in apply_presets:
+        rendering.update(given_presets[preset].rendering)
+        preset_models += given_presets[preset].models
+
+    assert 'rendering' in data
+    rendering.update(data['rendering'])
+    self.rendering = rendering
+
+    models = data.get('models')
+    self.models = preset_models
+    if models:
+      assert _is_list_of_strings(models)
+      assert all(m in existing_models for m in models)
+      self.models = set(models + self.models)
+
+  def to_filament_format(self):
+    json_out = {
+        'name': self.name,
+        'base': self.rendering
+    }
+    return json.dumps(json_out)
+
+class RenderTestConfig():
+  def __init__(self, data):
+    assert 'name' in data
+    name = data['name']
+    assert _is_string(name)
+    self.name = name
+
+    assert 'backends' in data
+    backends = data['backends']
+    assert _is_list_of_strings(backends)
+    self.backends = backends
+
+    assert 'model_search_paths' in data
+    model_search_paths = data.get('model_search_paths')
+    assert _is_list_of_strings(model_search_paths)
+    assert all(path.isdir(p) for p in model_search_paths)
+
+    model_paths = list(
+        chain(*(glob.glob(f'{d}/**/*.glb', recursive=True) for d in model_search_paths)))
+    # This flatten the output for glob.glob
+    self.models = {path.splitext(path.basename(model))[0]: model for model in model_paths}
+
+    preset_data = data.get('presets')
+    presets = []
+    if preset_data:
+      presets = [PresetConfig(p, self.models) for p in preset_data]
+
+    assert 'tests' in data
+    self.tests = [TestConfig(t, self.models, presets) for t in data['tests']]
+    test_names = list([t.name for t in self.tests])
+
+    # We cannot have duplicate test names
+    assert len(test_names) == len(set(test_names))
+
+def _remove_comments_from_json_txt(json_txt):
+  res = []
+  for line in json_txt.split('\n'):
+    if '//' in line:
+      line = line.split('//')[0]
+    res.append(line)
+  return '\n'.join(res)
+
+def parse_test_config_from_path(config_path):
+  with open(config_path, 'r') as f:
+    json_txt = json.loads(_remove_comments_from_json_txt(f.read()))
+    return RenderTestConfig(json_txt)
+
+
+if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--test', help='Configuration of the test', required=True)
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+  test = parse_test_config_from_path(args.test)

test/renderdiff/src/run.py

@@ -0,0 +1,111 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+
+from utils import execute, ArgParseImpl
+from parse_test_json import parse_test_config_from_path
+
+def important_print(msg):
+  lines = msg.split('\n')
+  max_len = max([len(l) for l in lines])
+  print('-' * (max_len + 8))
+  for line in lines:
+    diff = max_len - len(line)
+    information = f'--- {line} ' + (' ' * diff) + '---'
+    print(information)
+  print('-' * (max_len + 8))
+
+def render_test(gltf_viewer, test_config, output_dir,
+                opengl_lib=None, vk_icd=None):
+  assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
+  assert os.access(gltf_viewer, os.X_OK)
+
+  named_output_dir = os.path.join(output_dir, test_config.name)
+  execute(f'mkdir -p {named_output_dir}')
+
+  results = []
+  for test in test_config.tests:
+    test_json_path = f'{named_output_dir}/{test.name}.simplified.json'
+
+    with open(test_json_path, 'w') as f:
+      f.write(f'[{test.to_filament_format()}]')
+
+    for backend in test_config.backends:
+      env = None
+      if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib):
+        env = {'LD_LIBRARY_PATH': opengl_lib}
+
+      for model in test.models:
+        model_path = test_config.models[model]
+        out_name = f'{test.name}.{backend}.{model}'
+        test_desc = out_name
+
+        important_print(f'Rendering {test_desc}')
+
+        res, _ = execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
+                env=env, capture_output=False)
+
+        if res == 0:
+          execute(f'mv -f {test.name}0.ppm {named_output_dir}/{out_name}.ppm', capture_output=False)
+          execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', capture_output=False)
+        else:
+          important_print(f'{test_desc} failed with error={res}')
+        print('')
+
+        results.append((out_name, res))
+  return results
+
+GOLDENS_DIR = 'renderdiff_goldens'
+
+# We pull the goldens from the filament-assets repo
+def pull_goldens(output_dir):
+  assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
+  golden_dir = os.path.join(output_dir, "golden")
+  assets_dir = os.path.join(output_dir, "filament-assets")
+
+  if not os.path.exists(assets_dir):
+    execute('git clone --depth 1 git@github.com:google/filament-assets.git', cwd=output_dir)
+  else:
+    execute('git fetch', cwd=assets_dir)
+    execute('git checkout main ', cwd=assets_dir)
+    execute('git rebase', cwd=assets_dir)
+
+  if os.path.exists(golden_dir):
+    execute('rm -f goldens/*', cwd=output_dir)
+  execute(f'cp filament-assets/{GOLDENS_DIR}/* goldens', cwd=output_dir)
+
+def push_goldens(output_dir, test_name, filter_func=lambda a:True):
+  for test in test_config.tests:
+    for backend in test_config.backends:
+      for model in test.models:
+        pass
+
+if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--test', help='Configuration of the test', required=True)
+  parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True)
+  parser.add_argument('--output_dir', help='Output Directory', required=True)
+  parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
+  parser.add_argument('--vk_icd', help='Path to VK ICD file')
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+  test = parse_test_config_from_path(args.test)
+  render_result = render_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd)
+
+  failed = [f'   {tname}' for tname, res in render_result if res != 0]
+  success_count = len(render_result) - len(failed  )
+  important_print(f'Successfully rendered {success_count} / {len(render_result)}' +
+                  ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/utils.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+import argparse
+import sys
+
+def execute(cmd,
+            cwd=None,
+            capture_output=True,
+            stdin=None,
+            env=None,
+            raise_errors=False):
+  in_env = os.environ
+  in_env.update(env if env else {})
+  home = os.environ['HOME']
+  if f'{home}/bin' not in in_env['PATH']:
+    in_env['PATH'] = in_env['PATH'] + f':{home}/bin'
+
+  stdout = subprocess.PIPE if capture_output else sys.stdout
+  stderr = subprocess.PIPE if capture_output else sys.stdout
+  output = ''
+  err_output = ''
+  return_code = -1
+  kwargs = {
+      'cwd': cwd,
+      'env': in_env,
+      'stdout': stdout,
+      'stderr': stderr,
+      'stdin': stdin,
+      'universal_newlines': True
+  }
+  if capture_output:
+    process = subprocess.Popen(cmd.split(' '), **kwargs)
+    output, err_output = process.communicate()
+    return_code = process.returncode
+  else:
+    return_code = subprocess.call(cmd.split(' '), **kwargs)
+
+  if return_code:
+    # Error
+    if raise_errors:
+      raise subprocess.CalledProcessError(return_code, cmd)
+  if output:
+    if type(output) != str:
+      try:
+        output = output.decode('utf-8').strip()
+      except UnicodeDecodeError as e:
+        print('cannot decode ', output, file=sys.stderr)
+  return return_code, (output if return_code == 0 else err_output)
+
+class ArgParseImpl(argparse.ArgumentParser):
+  def error(self, message):
+    sys.stderr.write('error: %s\n' % message)
+    self.print_help()
+    sys.exit(1)

test/renderdiff/src/workflow_presubmit_msg.py

@@ -0,0 +1,46 @@
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from utils import execute
+
+def get_last_commit():
+  res, o = execute('git log -1')
+  commit, author, date, _, title, *desc = o.split('\n')
+
+  desc = [l.strip() for l in desc[1:]]
+  if len(desc) > 0 and len(desc[0]) == 0:
+    desc = desc[1:]
+
+  return (
+    commit.split(' ')[1],
+    title.strip(),
+    desc)
+
+def sanitized_split(line, split_atom='\n'):
+  return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom))))
+
+RDIFF_UPDATE_GOLDEN_STR = 'RDIFF_UPDATE_GOLDEN'
+
+if __name__ == "__main__":
+  RE_STR = f'{RDIFF_UPDATE_GOLDEN_STR}(?:S)?=[\[]?([a-zA-Z0-9,\s]+)[\]]?'
+
+  to_update = []
+  commit, title, description = get_last_commit()
+  for line in description:
+    m = re.match(RE_STR, line)
+    if not m:
+      continue
+
+    to_update += sanitize_split(m.group(1).replace(',', ' '), ' ')
+  print(','.join(to_update))