renderdiff: enable vulkan (#9108)

RDIFF_BRANCH=pf/renderdiff-enable-vulkan

.github/actions/get-vulkan-sdk/action.yml

@@ -8,8 +8,8 @@ runs:
       uses: actions/cache@v3
       id: cache-vulkan-sdk
       with:
-        path: vulkansdk
-        key: vulkansdk-${{ env.GITHUB_VULKANSDK_VERSION }}-${{ runner.os }}
+        path: ${{ runner.homedir }}/VulkanSDK
+        key: vulkansdk-${{ env.GITHUB_VULKANSDK_VERSION }}-2-${{ runner.os }}
     - name: Download Vulkan SDK
       if: steps.cache-vulkan-sdk.outputs.cache-hit != 'true'
       run: |
@@ -21,4 +21,8 @@ runs:
       run: |
         source ${{ github.workspace }}/build/common/get-vulkan-sdk.sh
         unpack_vulkan_installer
+        pushd .
+        cd ~/VulkanSDK/${GITHUB_VULKANSDK_VERSION}
+        sudo ./install_vulkan.py
+        popd
       shell: bash

.github/workflows/presubmit.yml

@@ -122,9 +122,12 @@ jobs:
         uses: ./.github/actions/get-commit-msg
       - uses: ./.github/actions/mac-prereq
       - uses: ./.github/actions/get-mesa
+      - uses: ./.github/actions/get-vulkan-sdk
       - name: Prerequisites
         run: |
           pip install tifffile numpy
+          # Must have at least clang-16 for a webgpu/dawn build.
+          sudo xcode-select -s /Applications/Xcode_16.2.app/Contents/Developer
         shell: bash
       - name: Render and compare
         id: render_compare

build/common/get-mesa.sh

@@ -137,6 +137,9 @@ if  [[ "$OS_NAME" == "Darwin" ]]; then
 
     # This is necessary to be able to build vk (lavapipe) on macOS.  Doesn't seem like a real dependency.
     sed -I '' "s/error('Vulkan drivers require dri3 for X11 support')//g" meson.build
+    # This is to properly link lib-xcb-present on the mac build (though we won't be drawing to any
+    # real hardware surface).
+    sed -I '' "s/dep_xcb_present = null_dep/dep_xcb_present = dependency('xcb-present')/g" meson.build
 fi
 
 # -Dosmesa=true    => builds OSMesa, which is an offscreen GL context

build/common/get-vulkan-sdk.sh

@@ -109,13 +109,8 @@ function install_mac() {
     unzip -t vulkan_sdk.zip
   fi
   echo "recognized zip layout 'vulkan_sdk.zip' ${InstallVulkan}.app/Contents" >&2
-  local sdk_temp=${VULKAN_SDK_DIR}.tmp
-  sudo ${InstallVulkan}.app/Contents/MacOS/${InstallVulkan} --root "$sdk_temp" --accept-licenses --default-answer --confirm-command install
-  du -hs $sdk_temp
-  test -d $sdk_temp/macOS || { echo "unrecognized dmg folder layout: $sdk_temp" ; ls -l $sdk_temp ; }
-  cp -r $sdk_temp/macOS/* $VULKAN_SDK_DIR/
+  ${InstallVulkan}.app/Contents/MacOS/${InstallVulkan} --accept-licenses --default-answer --confirm-command install
   if [[ -d ${InstallVulkan}.app/Contents ]] ; then
-    sudo rm -rf "$sdk_temp"
     rm -rf ${InstallVulkan}.app
   fi
 }

docs_src/README.md

@@ -65,6 +65,9 @@ We describe step 1 in detail for the sake of record:
  - Replace css styling in the exported output as needed (so they don't interfere with the book's css.
  - Replace resource urls to refer to locations relative to the mdbook structure.
 
+Any `markdeep` doc can be placed in `docs_src/src_markdeep/` and they will be parsed to html and included
+in the book as above.
+
 ### READMEs
 Filament depends on a number of libraries, which reside in the directory `libs`. These individual
 libaries often have README.md in their root to describe itself. We collect these descriptions into our
@@ -75,7 +78,7 @@ The process for copying and processing these READMEs is outlined in [Introductor
 
 ### Other technical notes
 These are technical documents that do not fit into a library, tool, or directory of the
-Filament source tree. We collect them into the `docs_src/src/notes` directory. No additional
+Filament source tree. We collect them into the `docs_src/src_mdbook/src/notes` directory. No additional
 processing is needed for these documents.
 
 ### Raw source files
@@ -90,7 +93,7 @@ add a link in `SUMMARY.md`, and perform the steps outlined in
 [how-to create section](#how-to-create).
 
 For example, if you are adding a general technical note, then you would
- - Place the document (file with extension `.md`) in `docs_src/src/notes`
+ - Place the document (file with extension `.md`) in `docs_src/src_mdbook/src/notes`
  - Add a link in [`docs_src/src_mdbook/src/SUMMARY.md`]
  - Run the commands in the [Generate](#how-to-generate) section
 

test/renderdiff/generate.sh

@@ -37,7 +37,7 @@ function start_render_() {
     # -W enables the webgpu build
     # -f forces regeneration of cmake build files
     # -X points to the mesa directory, which contains the compiled gl and vk drivers.
-    CXX=`which clang++` CC=`which clang` ./build.sh -W -f -X ${MESA_DIR} -p desktop debug gltf_viewer
+    CXX=`which clang++` CC=`which clang` ./build.sh -f -W -X ${MESA_DIR} -p desktop debug gltf_viewer
 }
 
 function end_render_() {
@@ -55,7 +55,8 @@ function end_render_() {
 start_render_ && \
     python3 ${RENDERDIFF_TEST_DIR}/src/render.py \
             --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \
-            --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \
-            --output_dir=${RENDER_OUTPUT_DIR} \
-            --opengl_lib=${MESA_LIB_DIR} && \
+            --test="${RENDERDIFF_TEST_DIR}/tests/presubmit.json" \
+            --output_dir="${RENDER_OUTPUT_DIR}" \
+            --opengl_lib="${MESA_LIB_DIR}" \
+            --vk_icd="${MESA_VK_ICD_PATH}" && \
     end_render_

test/renderdiff/src/commit_msg.py

@@ -41,7 +41,7 @@ def _parse_commit(commit_str):
   )
 
 if __name__ == "__main__":
-  RE_STR = rf"{RDIFF_UPDATE_GOLDEN_STR}(?:S)?=[\[]?([a-zA-Z0-9,\s\-\/]+)[\]]?"
+  RE_STR = rf"{RDIFF_UPDATE_GOLDEN_STR}(?:\s)?\=(?:\s)?([a-zA-Z0-9\s\-\/]+)"
 
   parser = ArgParseImpl()
   parser.add_argument('--file', help='A file containing the commit message')

test/renderdiff/src/compare.py

@@ -6,13 +6,34 @@ import json
 
 from utils import execute, ArgParseImpl, important_print, mkdir_p
 from image_diff import same_image, output_image_diff
-from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING
+from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING, GOLDEN_MISSING
 
 def _compare_goldens(base_dir, comparison_dir, out_dir=None):
   all_files = glob.glob(os.path.join(base_dir, "./**/*.tif"), recursive=True)
   test_dirs = set(os.path.abspath(os.path.dirname(f)).replace(os.path.abspath(base_dir) + '/', '') \
                   for f in all_files)
   all_results = []
+
+  def single_test(src_dir, dest_dir, src_fname):
+    src_fname = os.path.abspath(src_fname)
+    test_case = src_fname.replace(f'{src_dir}/', '')
+    dest_fname = os.path.join(dest_dir, test_case)
+    result = {
+      'name': test_case,
+    }
+    if not os.path.exists(dest_fname):
+      result['result'] = RESULT_MISSING
+    elif not same_image(src_fname, dest_fname):
+      result['result'] = RESULT_FAILED
+      if output_test_dir:
+        # just the file name
+        diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
+        output_image_diff(src_fname, dest_fname, os.path.join(output_test_dir, diff_fname))
+        result['diff'] = diff_fname
+    else:
+      result['result'] = RESULT_OK
+    return result
+
   for test_dir in test_dirs:
     results = []
     output_test_dir = None if not out_dir else os.path.abspath(os.path.join(out_dir, test_dir))
@@ -20,27 +41,23 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None):
       mkdir_p(output_test_dir)
     base_test_dir = os.path.abspath(os.path.join(base_dir, test_dir))
     comp_test_dir = os.path.abspath(os.path.join(comparison_dir, test_dir))
-    for golden_file in \
-        glob.glob(os.path.join(base_test_dir, "*.tif")):
-      base_fname = os.path.abspath(golden_file)
-      test_case = base_fname.replace(f'{base_test_dir}/', '')
-      comp_fname = os.path.join(comp_test_dir, test_case)
-      result = {
-        'name': test_case,
-      }
-      if not os.path.exists(comp_fname):
-        print(f'file name not found: {comp_fname}')
-        result['result'] = RESULT_MISSING
-      elif not same_image(base_fname, comp_fname):
-        result['result'] = RESULT_FAILED
-        if output_test_dir:
-          # just the file name
-          diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
-          output_image_diff(base_fname, comp_fname, os.path.join(output_test_dir, diff_fname))
-          result['diff'] = diff_fname
-      else:
-        result['result'] = RESULT_OK
-      results.append(result)
+    results = [
+      single_test(base_test_dir, comp_test_dir, golden_file) \
+      for golden_file in glob.glob(os.path.join(base_test_dir, "*.tif"))
+    ]
+    seen_test_cases = set([r['name'] for r in results])
+
+    # For files that are rendered but not in the golden directory
+    for base_file in \
+        glob.glob(os.path.join(comp_test_dir, "*.tif")):
+      src_fname = os.path.abspath(base_file)
+      test_case = base_file.replace(f'{comp_test_dir}/', '')
+      if test_case not in seen_test_cases:
+        results.append({
+          'name': test_case,
+          'result': GOLDEN_MISSING,
+        })
+
     if output_test_dir:
       output_fname = os.path.join(output_test_dir, "compare_results.json")
       results_meta = {
@@ -70,7 +87,7 @@ if __name__ == '__main__':
 
   results = _compare_goldens(args.src, dest, out_dir=args.out)
 
-  failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
+  failed = [f"   {k['name']} ({k['result']})" for k in results if k['result'] != RESULT_OK]
   success_count = len(results) - len(failed)
   important_print(f'Successfully compared {success_count} / {len(results)} images' +
                     ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/preamble.sh

@@ -29,6 +29,7 @@ if [[ "$os_name" == "Linux" ]]; then
     MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu"
 elif [[ "$os_name" == "Darwin" ]]; then
     MESA_LIB_DIR="${MESA_DIR}lib"
+    MESA_VK_ICD_PATH="${MESA_DIR}share/vulkan/icd.d/lvp_icd.aarch64.json"
 else
     echo "Unsupported platform for renderdiff tests"
     exit 1

test/renderdiff/src/render.py

@@ -26,15 +26,24 @@ from golden_manager import GoldenManager
 from image_diff import same_image
 from results import RESULT_OK, RESULT_FAILED
 
-def _render_single_model(gltf_viewer, test_json_path, named_output_dir, test_name, backend, model, model_path, opengl_lib):
-  env = None
+def _render_single_model(gltf_viewer, test_json_path, named_output_dir,
+                         test_name, backend, model, model_path, opengl_lib, vk_icd):
+  # We need to pass along the old environment because it might include set up from vulkansdk.
+  env = os.environ.copy()
   if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib):
-    env = {
+    env |= {
       'LD_LIBRARY_PATH': opengl_lib,
       # for macOS
       'DYLD_LIBRARY_PATH': opengl_lib,
     }
 
+  if backend == 'vulkan' and os.path.exists(vk_icd):
+    env |= {
+      'VK_ICD_FILENAMES': vk_icd,
+      'VK_DRIVER_FILES': vk_icd,
+      'VK_LOADER_DEBUG': 'all',
+    }
+
   out_name = f'{test_name}.{backend}.{model}'
   test_desc = out_name
 
@@ -43,10 +52,10 @@ def _render_single_model(gltf_viewer, test_json_path, named_output_dir, test_nam
 
   important_print(f'Rendering {test_desc}')
 
-  out_code, _ = execute(
+  out_code, output = execute(
     f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
     cwd=working_dir,
-    env=env, capture_output=False
+    env=env, capture_output=True
   )
 
   result = ''
@@ -56,9 +65,10 @@ def _render_single_model(gltf_viewer, test_json_path, named_output_dir, test_nam
     out_tif_name = f'{named_output_dir}/{out_tif_basename}'
     mv_f(f'{working_dir}/{test_name}0.tif', out_tif_name)
     mv_f(f'{working_dir}/{test_name}0.json', f'{named_output_dir}/{test_name}.json')
+    important_print(f'{test_desc} rendering succeeded. output=\n{output}')
   else:
     result = RESULT_FAILED
-    important_print(f'{test_desc} rendering failed with error={out_code}')
+    important_print(f'{test_desc} rendering failed with error={out_code}output=\n{output}')
 
   return {
     'name': out_name,
@@ -91,17 +101,18 @@ def _render_test_config(gltf_viewer,
         f.write(f'[{test.to_filament_format()}]')
 
       for backend in test_config.backends:
+        if backend == 'vulkan':
+          assert vk_icd, "VK ICD must be specified when testing vulkan backend"
         for model in test.models:
           model_path = os.path.abspath(test_config.models[model])
           futures.append(
             executor.submit(_render_single_model, gltf_viewer_abs,
                     test_json_path, named_output_dir,
                     test.name, backend, model, model_path,
-                    opengl_lib))
+                    opengl_lib, vk_icd))
 
     for future in concurrent.futures.as_completed(futures):
       results.append(future.result())
-  print(results)
 
   return named_output_dir, results
 

test/renderdiff/src/results.py

@@ -1,3 +1,4 @@
 RESULT_OK = 'ok'
 RESULT_FAILED = 'failed'
 RESULT_MISSING = 'missing'
+GOLDEN_MISSING = 'golden missing'

test/renderdiff/tests/presubmit.json

@@ -1,6 +1,6 @@
 {
     "name": "presubmit",
-    "backends": ["opengl"],
+    "backends": ["opengl", "vulkan"],
     "model_search_paths": ["third_party/models"],
     "presets": [
         {