filament/test/renderdiff/src/run.py

# Copyright (C) 2024 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import os
import json
import glob
import shutil

from utils import execute, ArgParseImpl, mkdir_p, mv_f
from parse_test_json import parse_test_config_from_path
from golden_manager import GoldenManager
from image_diff import same_image

def important_print(msg):
  lines = msg.split('\n')
  max_len = max([len(l) for l in lines])
  print('-' * (max_len + 8))
  for line in lines:
    diff = max_len - len(line)
    information = f'--- {line} ' + (' ' * diff) + '---'
    print(information)
  print('-' * (max_len + 8))

RESULT_OK = 'ok'
RESULT_FAILED_TO_RENDER = 'failed-to-render'
RESULT_FAILED_IMAGE_DIFF = 'failed-image-diff'
RESULT_FAILED_NO_GOLDEN = 'failed-no-golden'

def run_test(gltf_viewer,
             test_config,
             output_dir,
             opengl_lib=None,
             vk_icd=None):
  assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist"
  assert os.access(gltf_viewer, os.X_OK)

  named_output_dir = os.path.join(output_dir, test_config.name)
  mkdir_p(named_output_dir)

  results = []
  for test in test_config.tests:
    test_json_path = f'{named_output_dir}/{test.name}.simplified.json'

    with open(test_json_path, 'w') as f:
      f.write(f'[{test.to_filament_format()}]')

    for backend in test_config.backends:
      env = None
      if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib):
        env = {
          'LD_LIBRARY_PATH': opengl_lib,

           # for macOS
          'DYLD_LIBRARY_PATH': opengl_lib,
        }

      for model in test.models:
        model_path = test_config.models[model]
        out_name = f'{test.name}.{backend}.{model}'
        test_desc = out_name

        important_print(f'Rendering {test_desc}')

        out_code, _ = execute(
          f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
          env=env, capture_output=False
        )

        result = ''
        if out_code == 0:
          result = RESULT_OK
          out_tif_basename = f'{out_name}.tif'
          out_tif_name = f'{named_output_dir}/{out_tif_basename}'
          mv_f(f'{test.name}0.tif', out_tif_name)
          mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json')
        else:
          result = RESULT_FAILED_TO_RENDER
          important_print(f'{test_desc} rendering failed with error={out_code}')

        results.append({
          'name': out_name,
          'result': result,
          'result_code': out_code,
        })
  return named_output_dir, results

def compare_goldens(render_results, output_dir, goldens):
  for result in render_results:
    if result['result'] != RESULT_OK:
      continue

    out_tif_basename = f"{result['name']}.tif"
    out_tif_name = f'{output_dir}/{out_tif_basename}'
    golden_path = goldens.get(out_tif_basename)
    if not golden_path:
      result['result'] = RESULT_FAILED_NO_GOLDEN
      result['result_code'] = 1
    elif not same_image(golden_path, out_tif_name):
      result['result'] = RESULT_FAILED_IMAGE_DIFF
      result['result_code'] = 1

  return render_results

if __name__ == "__main__":
  parser = ArgParseImpl()
  parser.add_argument('--test', help='Configuration of the test', required=True)
  parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True)
  parser.add_argument('--output_dir', help='Output Directory', required=True)
  parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
  parser.add_argument('--vk_icd', help='Path to VK ICD file')
  parser.add_argument('--golden_branch', help='Branch of the golden repo to compare against')

  args, _ = parser.parse_known_args(sys.argv[1:])
  test = parse_test_config_from_path(args.test)

  output_dir, results = \
    run_test(args.gltf_viewer,
             test,
             args.output_dir,
             opengl_lib=args.opengl_lib,
             vk_icd=args.vk_icd)

  do_compare = False
  # The presence of this argument indicates comparison against a set of goldens.
  if args.golden_branch:
    # prepare goldens working directory
    tmp_golden_dir = '/tmp/renderdiff-goldens'
    mkdir_p(tmp_golden_dir)

    # Download the golden repo into the current working directory
    golden_manager = GoldenManager(os.getcwd())
    golden_manager.download_to(tmp_golden_dir, branch=args.golden_branch)

    goldens = {
      os.path.basename(fpath) : fpath for fpath in \
      glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
    }
    results = compare_goldens(results, output_dir, goldens)
    do_compare = True

  with open(f'{output_dir}/results.json', 'w') as f:
    f.write(json.dumps(results))

  shutil.copy2(args.test, f'{output_dir}/test.json')

  failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
  success_count = len(results) - len(failed)
  op = 'tested' if do_compare else 'rendered'
  important_print(f'Successfully {op} {success_count} / {len(results)}' +
                  ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))