github: add software rasterizer job for GL to presubmit (#8158)

We use Mesa's gallium swrast to render as the driver with
Filament's backend set to GL. We provide a few scripts to parse
the tests (as jsons) and run gltf_viewer to produce the rendering.

.github/actions/ubuntu-apt-add-src/action.yml

@@ -0,0 +1,9 @@
+name: 'ubuntu apt add deb-src'
+runs:
+  using: "composite"
+  steps:
+    - name: "ubuntu apt add deb-src"
+      run: |
+          echo "deb-src http://archive.ubuntu.com/ubuntu jammy main restricted universe" | sudo tee /etc/apt/sources.list.d/my.list
+          sudo apt-get update
+      shell: bash

.github/workflows/presubmit.yml

@@ -76,3 +76,18 @@ jobs:
       - name: Run build script
         run: |
           cd build/web && printf "y" | ./build.sh presubmit
+
+  test-renderdiff:
+    name: test-renderdiff
+    runs-on: ubuntu-22.04-32core
+
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: ./.github/actions/ubuntu-apt-add-src
+      - name: Run script
+        run: |
+          source ./build/linux/ci-common.sh && bash test/renderdiff_tests.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: presubmit-renderdiff-result
+          path: ./out/renderdiff_tests

test/renderdiff/README.md

@@ -0,0 +1,17 @@
+# Rendering Difference Test
+
+We created a few scripts to run `gltf_viewer` and produce headless renderings.
+
+This is mainly useful for continuous integration where GPUs are generally not available on cloud
+machines. To perform software rasterization, these scripts are centered around [Mesa]'s software
+rasterizers, but nothing bars us from using another rasterizer like [SwiftShader]. Additionally,
+we should be able to use GPUs where available (though this is more of a future work).
+
+The script `run.py` contains the core logic for taking input parameters (such as the test
+description file) and then running gltf_viewer to produce the results.
+
+In the `test` directory is a list of test descriptions that are specified in json.  Please see
+`sample.json` to parse the structure.
+
+[Mesa]: https://docs.mesa3d.org
+[SwiftShader]: https://github.com/google/swiftshader

test/renderdiff/parse_test_json.py

@@ -0,0 +1,142 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from utils import execute, ArgParseImpl
+
+import glob
+from itertools import chain
+import json
+import sys
+import os
+from os import path
+
+def _is_list_of_strings(field):
+  return isinstance(field, list) and\
+      all(isinstance(item, str) for item in field)
+
+def _is_string(s):
+    return isinstance(s, str)
+
+def _is_dict(s):
+    return isinstance(s, dict)
+
+class RenderingConfig():
+  def __init__(self, data):
+    assert 'name' in data
+    assert _is_string(data['name'])
+    self.name = data['name']
+
+    assert 'rendering' in data
+    assert _is_dict(data['rendering'])
+    self.rendering = data['rendering']
+
+class PresetConfig(RenderingConfig):
+  def __init__(self, data, existing_models):
+    RenderingConfig.__init__(self, data)
+    models = data.get('models')
+    if models:
+      assert _is_list_of_strings(models)
+      assert all(m in existing_models for m in models)
+      self.models = models
+
+class TestConfig(RenderingConfig):
+  def __init__(self, data, existing_models, presets):
+    RenderingConfig.__init__(self, data)
+    description = data.get('description')
+    if description:
+      assert _is_string(description)
+      self.description = description
+
+    apply_presets = data.get('apply_presets')
+    rendering = {}
+    preset_models = []
+    if apply_presets:
+      given_presets = {p.name: p for p in presets}
+      assert all((name in given_presets) for name in apply_presets)
+      for preset in apply_presets:
+        rendering.update(given_presets[preset].rendering)
+        preset_models += given_presets[preset].models
+
+    assert 'rendering' in data
+    rendering.update(data['rendering'])
+    self.rendering = rendering
+
+    models = data.get('models')
+    self.models = preset_models
+    if models:
+      assert _is_list_of_strings(models)
+      assert all(m in existing_models for m in models)
+      self.models = set(models + self.models)
+
+  def to_filament_format(self):
+    json_out = {
+        'name': self.name,
+        'base': self.rendering
+    }
+    return json.dumps(json_out)
+
+class RenderTestConfig():
+  def __init__(self, data):
+    assert 'name' in data
+    name = data['name']
+    assert _is_string(name)
+    self.name = name
+
+    assert 'backends' in data
+    backends = data['backends']
+    assert _is_list_of_strings(backends)
+    self.backends = backends
+
+    assert 'model_search_paths' in data
+    model_search_paths = data.get('model_search_paths')
+    assert _is_list_of_strings(model_search_paths)
+    assert all(path.isdir(p) for p in model_search_paths)
+
+    model_paths = list(
+        chain(*(glob.glob(f'{d}/**/*.glb', recursive=True) for d in model_search_paths)))
+    # This flatten the output for glob.glob
+    self.models = {path.splitext(path.basename(model))[0]: model for model in model_paths}
+
+    preset_data = data.get('presets')
+    presets = []
+    if preset_data:
+      presets = [PresetConfig(p, self.models) for p in preset_data]
+
+    assert 'tests' in data
+    self.tests = [TestConfig(t, self.models, presets) for t in data['tests']]
+    test_names = list([t.name for t in self.tests])
+
+    # We cannot have duplicate test names
+    assert len(test_names) == len(set(test_names))
+
+def _remove_comments_from_json_txt(json_txt):
+  res = []
+  for line in json_txt.split('\n'):
+    if '//' in line:
+      line = line.split('//')[0]
+    res.append(line)
+  return '\n'.join(res)
+
+def parse_test_config_from_path(config_path):
+  with open(config_path, 'r') as f:
+    json_txt = json.loads(_remove_comments_from_json_txt(f.read()))
+    return RenderTestConfig(json_txt)
+
+
+if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--test', help='Configuration of the test', required=True)
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+  test = parse_test_config_from_path(args.test)

test/renderdiff/run.py

@@ -0,0 +1,54 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from utils import execute, ArgParseImpl
+
+from parse_test_json import parse_test_config_from_path
+import sys
+import os
+
+def run_test(gltf_viewer, pixel_test, output_dir, opengl_lib=None, vk_icd=None):
+  assert os.path.isdir(output_dir)
+  assert os.access(gltf_viewer, os.X_OK)
+
+  for test in pixel_test.tests:
+    test_json_path = f'{output_dir}/{test.name}_simplified.json'
+
+    with open(test_json_path, 'w') as f:
+      f.write(f'[{test.to_filament_format()}]')
+
+    for backend in pixel_test.backends:
+      env = None
+      if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib):
+        env = {'LD_LIBRARY_PATH': opengl_lib}
+
+      for model in test.models:
+        model_path = pixel_test.models[model]
+        out_name = f'{test.name}_{model}_{backend}'
+        execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless',
+                env=env, capture_output=False)
+        execute(f'mv -f {test.name}0.ppm {output_dir}/{out_name}.ppm', capture_output=False)
+        execute(f'mv -f {test.name}0.json {output_dir}/{test.name}.json', capture_output=False)
+
+if __name__ == "__main__":
+  parser = ArgParseImpl()
+  parser.add_argument('--test', help='Configuration of the test', required=True)
+  parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True)
+  parser.add_argument('--output_dir', help='Output Directory', required=True)
+  parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)')
+  parser.add_argument('--vk_icd', help='Path to VK ICD file')
+
+  args, _ = parser.parse_known_args(sys.argv[1:])
+  test = parse_test_config_from_path(args.test)
+  run_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd)

test/renderdiff/tests/presubmit.json

@@ -0,0 +1,34 @@
+{
+    "name": "PresubmitPixelTests",
+    "backends": ["opengl"],
+    "model_search_paths": ["third_party/models"],
+    "presets": [
+        {
+            "name": "Standard",
+            "models": ["lucy", "DamagedHelmet"],
+            "rendering": {
+                "viewer.cameraFocusDistance": 0,
+                "view.postProcessingEnabled": true
+            }
+        }
+    ],
+    "tests": [
+        {
+            "name": "BloomFlare",
+            "description": "Testing bloom and flare",
+            "apply_presets": ["Standard"],
+            "rendering": {
+                "view.bloom.enabled": true,
+                "view.bloom.lensFlare": true
+            }
+        },
+        {
+            "name": "MSAA",
+            "description": "Testing Multi-sample Anti-aliasing",
+            "apply_presets": ["Standard"],
+            "rendering": {
+                "view.msaa.enabled": true
+            }
+        }
+    ]
+}

test/renderdiff/tests/sample.json

@@ -0,0 +1,34 @@
+{
+    "name": "SampleTest" ,                            // [required]
+    "backends": ["opengl"],                           // [required] Specifies the backend that will be used to render
+                                                      //     this test
+    "model_search_paths": ["third_party/models"],     // [optional] Base iirectory from which we will glob for
+                                                      //     .glb files and try to match against names in the 'models'
+                                                      //     field.
+    "presets": [                                      // [optional] A list of preset configurations that tests can
+                                                      //     inherit from.
+        {
+            "name": "Standard",                       // [required]
+            "models": ["lucy", "DamagedHelmet"],      // [optional] Base name for the gltf file used in the test.  For
+                                                      //     example, source files are lucy.glb and DamagedHelmet.gltf
+            "rendering": {                            // [required] Rendering settings used in the test. The json format
+                "viewer.cameraFocusDistance": 0,      //     is taken from AutomationSpec in libs/viewer
+                "view.postProcessingEnabled": true
+            }
+        }
+    ],
+    "tests": [                                        // [required] List of test configurations
+        {
+            "name": "BloomFlare",                     // [required]
+            "description": "Testing bloom and flare", // [optional]
+            "apply_presets": ["Standard"],            // [optional] Applies the preset in order. Item must be in
+                                                      //     'presets' field in the top-level struct.
+            "model": [],                              // [optional] List of models used in this test.  The list is
+                                                      //      *appended* onto the lists provided by the presets.
+            "rendering": {                            // [required] See description in 'presets'
+                "view.bloom.enabled": true,
+                "view.bloom.lensFlare": true
+            }
+        }
+    ]
+}

test/renderdiff/utils.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+import argparse
+import sys
+
+def execute(cmd,
+            cwd=None,
+            capture_output=True,
+            stdin=None,
+            env=None,
+            raise_errors=False):
+  in_env = os.environ
+  in_env.update(env if env else {})
+  home = os.environ['HOME']
+  if f'{home}/bin' not in in_env['PATH']:
+    in_env['PATH'] = in_env['PATH'] + f':{home}/bin'
+
+  stdout = subprocess.PIPE if capture_output else sys.stdout
+  stderr = subprocess.PIPE if capture_output else sys.stdout
+  output = ''
+  err_output = ''
+  return_code = -1
+  kwargs = {
+      'cwd': cwd,
+      'env': in_env,
+      'stdout': stdout,
+      'stderr': stderr,
+      'stdin': stdin,
+      'universal_newlines': True
+  }
+  if capture_output:
+    process = subprocess.Popen(cmd.split(' '), **kwargs)
+    output, err_output = process.communicate()
+    return_code = process.returncode
+  else:
+    return_code = subprocess.call(cmd.split(' '), **kwargs)
+
+  if return_code:
+    # Error
+    if raise_errors:
+      raise subprocess.CalledProcessError(return_code, cmd)
+  if output:
+    if type(output) != str:
+      try:
+        output = output.decode('utf-8').strip()
+      except UnicodeDecodeError as e:
+        print('cannot decode ', output, file=sys.stderr)
+  return return_code, (output if return_code == 0 else err_output)
+
+class ArgParseImpl(argparse.ArgumentParser):
+  def error(self, message):
+    sys.stderr.write('error: %s\n' % message)
+    self.print_help()
+    sys.exit(1)

test/renderdiff_tests.sh

@@ -0,0 +1,45 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/bash
+
+OUTPUT_DIR="$(pwd)/out/renderdiff_tests"
+RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff"
+TEST_UTILS_DIR="$(pwd)/test/utils"
+MESA_DIR="$(pwd)/mesa/out/"
+MESA_LIB_DIR="${MESA_DIR}/lib/x86_64-linux-gnu"
+
+function prepare_mesa() {
+    if [ ! -d ${MESA_LIB_DIR} ]; then
+        rm -rf mesa
+        bash ${TEST_UTILS_DIR}/get_mesa.sh
+    fi
+}
+
+# Following steps are taken:
+#  - Get and build mesa
+#  - Build gltf_viewer
+#  - Run the python script that runs the test
+#  - Zip up the result
+
+set -e && set -x && prepare_mesa && \
+    mkdir -p ${OUTPUT_DIR} && \
+    ./build.sh -X ${MESA_DIR} -p desktop debug gltf_viewer && \
+    python3 ${RENDERDIFF_TEST_DIR}/run.py \
+            --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \
+            --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \
+            --output_dir=${OUTPUT_DIR} \
+            --opengl_lib=${MESA_LIB_DIR}
+
+unset MESA_LIB_DIR

test/utils/get_mesa.sh

@@ -0,0 +1,38 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/bash
+
+set -e
+set -x
+
+sudo apt-get -y build-dep mesa
+
+git clone https://gitlab.freedesktop.org/mesa/mesa.git
+
+pushd .
+
+cd mesa
+
+git checkout mesa-23.2.1
+
+mkdir -p out
+
+# -Dosmesa=true    => builds OSMesa, which is an offscreen GL context
+# -Dgallium-drivers=swrast  => builds GL software rasterizer
+# -Dvulkan-drivers=swrast   => builds VK software rasterizer
+meson setup builddir/ -Dprefix="$(pwd)/out" -Dosmesa=true -Dglx=xlib -Dgallium-drivers=swrast -Dvulkan-drivers=swrast
+meson install -C builddir/
+
+popd