diff --git a/test/renderdiff/local_test.sh b/test/renderdiff/local_test.sh index 16b1362d64..89de0cdbb6 100755 --- a/test/renderdiff/local_test.sh +++ b/test/renderdiff/local_test.sh @@ -29,9 +29,11 @@ bash `dirname $0`/generate.sh "$@" && \ python3 ${RENDERDIFF_TEST_DIR}/src/golden_manager.py \ --branch=${GOLDEN_BRANCH} \ --output=${GOLDEN_OUTPUT_DIR} && \ - # Pass arguments to compare.py, e.g. --test_filter python3 ${RENDERDIFF_TEST_DIR}/src/compare.py \ --src=${GOLDEN_OUTPUT_DIR} \ --dest=${RENDER_OUTPUT_DIR} \ - --out=${DIFF_OUTPUT_DIR} "$@" + --out=${DIFF_OUTPUT_DIR} \ + --test="${RENDERDIFF_TEST_DIR}/tests/presubmit.json" "$@" + + # $@ Pass arguments to compare.py, e.g. --test_filter end_ diff --git a/test/renderdiff/src/compare.py b/test/renderdiff/src/compare.py index dfbe5bbeb1..7512c99f80 100644 --- a/test/renderdiff/src/compare.py +++ b/test/renderdiff/src/compare.py @@ -8,8 +8,81 @@ import fnmatch from utils import execute, ArgParseImpl, important_print, mkdir_p from image_diff import same_image, output_image_diff from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING, GOLDEN_MISSING +import test_config -def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None): +def _get_tolerance_for_test_case(test_case_name, test_config_obj): + if not test_config_obj: + return None + + # Extract test name from test case (remove backend and model) + # Format: TestName.backend.model -> TestName + test_name = test_case_name.split('.')[0] + + for test in test_config_obj.tests: + if test.name == test_name: + return test.tolerance + + return None + +def _format_tolerance_summary(stats): + """ + Create human-readable summary of tolerance statistics. + + Args: + stats: Statistics dictionary from tolerance evaluation + + Returns: + str: Formatted summary string + """ + if 'error' in stats: + return f"Error: {stats['error']}" + + if 'operator' in stats: + # Nested criteria with operator + operator = stats['operator'] + criteria_count = len(stats['criteria_results']) + passed_count = sum(1 for c in stats['criteria_results'] if c.get('passed', False)) + summary = f"{operator} of {criteria_count} criteria: {passed_count} passed, {criteria_count - passed_count} failed" + + # Add details for each criteria + details = [] + for i, criteria_stats in enumerate(stats['criteria_results']): + details.append(f" Criteria {i+1}: {_format_tolerance_summary(criteria_stats)}") + + return summary + "\n" + "\n".join(details) + else: + # Single criteria + total_pixels = stats.get('total_pixels', 0) + failing_pixels = stats.get('failing_pixels', 0) + failing_percentage = stats.get('failing_percentage', 0.0) + allowed_percentage = stats.get('allowed_percentage', 0.0) + max_abs_diff = stats.get('max_abs_diff', 0) + mean_abs_diff = stats.get('mean_abs_diff', 0) + max_diff_per_channel = stats.get('max_diff_per_channel', []) + + criteria = stats.get('criteria', {}) + criteria_desc = [] + if 'max_pixel_diff' in criteria: + criteria_desc.append(f"max_pixel_diff: {criteria['max_pixel_diff']}") + if 'max_pixel_diff_percent' in criteria: + criteria_desc.append(f"max_pixel_diff_percent: {criteria['max_pixel_diff_percent']}%") + if 'allowed_diff_pixels' in criteria: + criteria_desc.append(f"allowed_diff_pixels: {criteria['allowed_diff_pixels']}%") + + summary_lines = [ + f"Tolerance: {', '.join(criteria_desc)}", + f"Pixels: {failing_pixels:,} / {total_pixels:,} ({failing_percentage:.2f}%) exceed tolerance", + f"Allowed: {allowed_percentage:.2f}% - {'PASS' if stats.get('passed', False) else 'FAIL'}", + f"Max difference: {max_abs_diff} (mean: {mean_abs_diff:.1f})" + ] + + if len(max_diff_per_channel) > 1: + channel_info = ", ".join(f"Ch{i}: {diff}" for i, diff in enumerate(max_diff_per_channel)) + summary_lines.append(f"Per-channel max: {channel_info}") + + return "\n".join(summary_lines) + +def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None, test_config_path=None): def test_name(p): return p.replace('.tif', '') @@ -20,6 +93,14 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None): for f in all_files) all_results = [] + # Parse test configuration if provided + test_config_obj = None + if test_config_path and os.path.exists(test_config_path): + try: + test_config_obj = test_config.parse_from_path(test_config_path) + except Exception as e: + important_print(f"Warning: Could not parse test config {test_config_path}: {e}") + def single_test(src_dir, dest_dir, src_fname): src_fname = os.path.abspath(src_fname) test_case = src_fname.replace(f'{src_dir}/', '') @@ -27,17 +108,41 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None): result = { 'name': test_case, } + if not os.path.exists(dest_fname): result['result'] = RESULT_MISSING - elif not same_image(src_fname, dest_fname): - result['result'] = RESULT_FAILED - if output_test_dir: - # just the file name - diff_fname = f"{test_case.replace('.tif', '_diff.tif')}" - output_image_diff(src_fname, dest_fname, os.path.join(output_test_dir, diff_fname)) - result['diff'] = diff_fname else: - result['result'] = RESULT_OK + # Get tolerance configuration for this test case + tolerance = _get_tolerance_for_test_case(test_case.replace('.tif', ''), test_config_obj) + + # Compare images and get detailed statistics + comparison_result, stats = same_image(src_fname, dest_fname, tolerance) + + if not comparison_result: + result['result'] = RESULT_FAILED + if output_test_dir: + # just the file name + diff_fname = f"{test_case.replace('.tif', '_diff.tif')}" + output_image_diff(src_fname, dest_fname, os.path.join(output_test_dir, diff_fname)) + result['diff'] = diff_fname + else: + result['result'] = RESULT_OK + + # Add detailed tolerance information to result + if tolerance: + result['tolerance_used'] = True + result['tolerance_config'] = tolerance + if stats: + result['tolerance_stats'] = stats + # Add human-readable summary + result['tolerance_summary'] = _format_tolerance_summary(stats) + elif stats is None and comparison_result: + result['comparison_type'] = 'exact_match' + elif stats and 'error' in stats: + result['error'] = stats['error'] + if 'details' in stats: + result['error_details'] = stats['details'] + return result for test_dir in test_dirs: @@ -87,6 +192,7 @@ if __name__ == '__main__': parser.add_argument('--dest', help='Directory of the comparison of the diff.') parser.add_argument('--out', help='Directory of output for the result of the diff.') parser.add_argument('--test_filter', help='Filter for the tests to run') + parser.add_argument('--test', help='Path to test configuration JSON file for tolerance settings.') args, _ = parser.parse_known_args(sys.argv[1:]) @@ -96,11 +202,42 @@ if __name__ == '__main__': dest = os.path.join(os.getcwd(), './out/renderdiff') assert os.path.exists(dest), f"Destination folder={dest} does not exist." - results = _compare_goldens(args.src, dest, out_dir=args.out, test_filter=args.test_filter) + results = _compare_goldens(args.src, dest, out_dir=args.out, + test_filter=args.test_filter, test_config_path=args.test) - failed = [f" {k['name']} ({k['result']})" for k in results if k['result'] != RESULT_OK] - success_count = len(results) - len(failed) - important_print(f'Successfully compared {success_count} / {len(results)} images' + - ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) + # Categorize results + failed = [k for k in results if k['result'] != RESULT_OK] + passed = [k for k in results if k['result'] == RESULT_OK] + tolerance_used_count = len([k for k in results if k.get('tolerance_used', False)]) + + # Create detailed failure report + failed_details = [] + for k in failed: + failure_line = f" {k['name']} ({k['result']})" + if 'tolerance_summary' in k: + failure_line += f"\n {k['tolerance_summary'].replace(chr(10), chr(10) + ' ')}" + failed_details.append(failure_line) + + # Create success report with tolerance details + tolerance_used_details = [] + for k in passed: + if k.get('tolerance_used', False) and 'tolerance_summary' in k: + tolerance_used_details.append(f" {k['name']}: {k['tolerance_summary'].split(chr(10))[0]}") + + # Main summary + success_count = len(passed) + important_print(f'Successfully compared {success_count} / {len(results)} images') + + if tolerance_used_details: + pstr = 'Tolerance-based passes:' + for detail in tolerance_used_details: + pstr += '\n' + detail + important_print(pstr) + + if failed_details: + pstr = 'Failed:' + for detail in failed_details: + pstr = '\n' + detail + important_print(pstr) if len(failed) > 0: exit(1) diff --git a/test/renderdiff/src/image_diff.py b/test/renderdiff/src/image_diff.py index 4675c20b43..13f21a3502 100644 --- a/test/renderdiff/src/image_diff.py +++ b/test/renderdiff/src/image_diff.py @@ -15,7 +15,144 @@ import tifffile import numpy as np -def same_image(tiff_file_a, tiff_file_b): +def evaluate_tolerance_criteria(img1_data, img2_data, tolerance_spec): + """ + Recursively evaluate tolerance criteria with AND/OR logic. + + Args: + img1_data: First image array + img2_data: Second image array + tolerance_spec: Dictionary containing tolerance criteria with structure: + { + "operator": "AND" | "OR", # How to combine criteria results + "criteria": [...] # List of criteria or nested tolerance specs + } + OR for leaf criteria: + { + "max_pixel_diff": int, # Max absolute difference per channel (0-255) + "max_pixel_diff_percent": float, # Max difference as percentage (0-100%) + "allowed_diff_pixels": float # Percentage of pixels allowed to exceed (0-100%) + } + + Returns: + tuple: (bool: pass/fail, dict: detailed statistics) + """ + + if 'criteria' not in tolerance_spec: + # Leaf criteria - evaluate single condition + return evaluate_single_criteria(img1_data, img2_data, tolerance_spec) + + operator = tolerance_spec.get('operator', 'AND').upper() + criteria_list = tolerance_spec['criteria'] + + results_and_stats = [evaluate_tolerance_criteria(img1_data, img2_data, criteria) + for criteria in criteria_list] + + results = [r[0] for r in results_and_stats] # Extract pass/fail results + all_stats = [r[1] for r in results_and_stats] # Extract statistics + + if operator == 'AND': + final_result = all(results) + elif operator == 'OR': + final_result = any(results) + else: + raise ValueError(f"Unknown operator: {operator}") + + # Combine statistics + combined_stats = { + 'operator': operator, + 'criteria_results': all_stats, + 'passed': bool(final_result) + } + + return final_result, combined_stats + +def evaluate_single_criteria(img1_data, img2_data, criteria): + """ + Evaluate a single tolerance criteria. + + Args: + img1_data: First image array + img2_data: Second image array + criteria: Dictionary with tolerance parameters: + - max_pixel_diff: Maximum absolute difference per channel (0-255 range) + - max_pixel_diff_percent: Maximum difference as percentage (0-100%) + - allowed_diff_pixels: Percentage of pixels allowed to exceed tolerance (0-100%) + + Returns: + tuple: (bool: pass/fail, dict: detailed statistics) + """ + diff_abs = np.abs(img1_data.astype(np.int16) - img2_data.astype(np.int16)) + + max_diff = criteria.get('max_pixel_diff', float('inf')) + max_diff_percent = criteria.get('max_pixel_diff_percent', float('inf')) + allowed_diff_pixels = criteria.get('allowed_diff_pixels', 0.0) + + # Calculate which pixels exceed absolute threshold + exceeds_abs = diff_abs > max_diff if max_diff < float('inf') else np.zeros_like(diff_abs, dtype=bool) + + # Calculate which pixels exceed percentage threshold + if max_diff_percent < float('inf'): + max_val = np.maximum(img1_data, img2_data) + # Avoid division by zero + diff_percent = np.divide(diff_abs * 100.0, np.maximum(max_val, 1), + out=np.zeros_like(diff_abs, dtype=np.float32), + where=max_val!=0) + exceeds_percent = diff_percent > max_diff_percent + else: + exceeds_percent = np.zeros_like(diff_abs, dtype=bool) + + # A pixel fails if it exceeds either threshold (OR logic at pixel level) + exceeds_tolerance = exceeds_abs | exceeds_percent + + # Check per-pixel: pixel fails if ANY channel exceeds tolerance + exceeds_per_pixel = np.any(exceeds_tolerance, axis=-1) if len(exceeds_tolerance.shape) > 2 else exceeds_tolerance + + # Calculate detailed statistics + total_pixels = exceeds_per_pixel.size + failing_pixels = np.sum(exceeds_per_pixel) + failing_percentage = (failing_pixels / total_pixels) * 100.0 + + # Distribution analysis + max_abs_diff = np.max(diff_abs) if diff_abs.size > 0 else 0 + mean_abs_diff = np.mean(diff_abs) if diff_abs.size > 0 else 0 + + # Per-channel max differences + if len(diff_abs.shape) > 2: + max_diff_per_channel = [np.max(diff_abs[:, :, c]) for c in range(diff_abs.shape[2])] + else: + max_diff_per_channel = [max_abs_diff] + + stats = { + 'criteria': criteria, + 'total_pixels': int(total_pixels), + 'failing_pixels': int(failing_pixels), + 'failing_percentage': float(failing_percentage), + 'allowed_percentage': float(allowed_diff_pixels), + 'max_abs_diff': int(max_abs_diff), + 'mean_abs_diff': float(mean_abs_diff), + 'max_diff_per_channel': [int(x) for x in max_diff_per_channel], + 'passed': bool(failing_percentage <= allowed_diff_pixels) + } + + return failing_percentage <= allowed_diff_pixels, stats + +def same_image(tiff_file_a, tiff_file_b, tolerance=None): + """ + Compare two TIFF images for equality with optional tolerance. + + Args: + tiff_file_a: Path to first TIFF file + tiff_file_b: Path to second TIFF file + tolerance: Optional tolerance specification dictionary. If None, performs exact comparison. + Can be either: + 1. Single criteria: {"max_pixel_diff": 5, "allowed_diff_pixels": 1.0} + 2. Nested criteria: {"operator": "OR", "criteria": [...]} + + Returns: + tuple: (bool: pass/fail, dict: detailed statistics or None) + For exact comparison, returns (bool, None) + """ try: img1_data = tifffile.imread(tiff_file_a) img2_data = tifffile.imread(tiff_file_b) @@ -24,23 +161,25 @@ def same_image(tiff_file_a, tiff_file_b): # are different, the images are not the same. if img1_data.shape != img2_data.shape: print(f"Images have different shapes: {img1_data.shape} vs {img2_data.shape}") - return False + return False, {'error': 'Shape mismatch', 'shape1': img1_data.shape, 'shape2': img2_data.shape} - # numpy.array_equal() checks if two arrays have the same shape and elements. - if np.array_equal(img1_data, img2_data): - return True - else: - return False + # If no tolerance specified, use exact comparison + if tolerance is None: + exact_match = np.array_equal(img1_data, img2_data) + return exact_match, None + + # Use tolerance-based comparison + return evaluate_tolerance_criteria(img1_data, img2_data, tolerance) except FileNotFoundError: - print(f"Error: One or both files not found ('{file_path1}', '{file_path2}').") - return False + print(f"Error: One or both files not found ('{tiff_file_a}', '{tiff_file_b}').") + return False, {'error': 'File not found'} except tifffile.TiffFileError as e: print(f"Error: One or both files are not valid TIFF files or could not be read. Details: {e}") - return False + return False, {'error': 'Invalid TIFF file', 'details': str(e)} except Exception as e: print(f"An unexpected error occurred: {e}") - return False + return False, {'error': 'Unexpected error', 'details': str(e)} def output_image_diff(tiff_file_a, tiff_file_b, output_path): try: diff --git a/test/renderdiff/src/preamble.sh b/test/renderdiff/src/preamble.sh index e58330f98c..eac6e392c7 100644 --- a/test/renderdiff/src/preamble.sh +++ b/test/renderdiff/src/preamble.sh @@ -28,6 +28,7 @@ BUILD_COMMON_DIR="$(pwd)/build/common" os_name=$(uname -s) if [[ "$os_name" == "Linux" ]]; then MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu" + MESA_VK_ICD_PATH="${MESA_DIR}share/vulkan/icd.d/lvp_icd.x86_64.json" elif [[ "$os_name" == "Darwin" ]]; then MESA_LIB_DIR="${MESA_DIR}lib" MESA_VK_ICD_PATH="${MESA_DIR}share/vulkan/icd.d/lvp_icd.aarch64.json" diff --git a/test/renderdiff/src/test_config.py b/test/renderdiff/src/test_config.py index 1e15e194ef..b5f5d16da2 100644 --- a/test/renderdiff/src/test_config.py +++ b/test/renderdiff/src/test_config.py @@ -61,6 +61,54 @@ class PresetConfig(RenderingConfig): _check(models) self.models += models + # Parse tolerance configuration from preset + tolerance = data.get('tolerance') + if tolerance: + assert _is_dict(tolerance) + self._validate_tolerance(tolerance) + self.tolerance = tolerance + else: + self.tolerance = None + + def _validate_tolerance(self, tolerance): + """ + Validate tolerance configuration structure. + + Tolerance can be: + 1. Single criteria: {"max_pixel_diff": 5, "allowed_diff_pixels": 1.0} + 2. Nested criteria: {"operator": "OR", "criteria": [...]} + """ + if 'criteria' in tolerance: + # Nested structure with operator + operator = tolerance.get('operator', 'AND') + assert operator.upper() in ['AND', 'OR'], f"Invalid operator: {operator}" + + criteria_list = tolerance['criteria'] + assert isinstance(criteria_list, list), "criteria must be a list" + assert len(criteria_list) > 0, "criteria list cannot be empty" + + # Recursively validate each criteria + for criteria in criteria_list: + self._validate_tolerance(criteria) + else: + # Leaf criteria - validate individual parameters + valid_keys = {'max_pixel_diff', 'max_pixel_diff_percent', 'allowed_diff_pixels'} + tolerance_keys = set(tolerance.keys()) + invalid_keys = tolerance_keys - valid_keys + assert len(invalid_keys) == 0, f"Invalid tolerance keys: {invalid_keys}" + + if 'max_pixel_diff' in tolerance: + assert isinstance(tolerance['max_pixel_diff'], (int, float)), "max_pixel_diff must be numeric" + assert 0 <= tolerance['max_pixel_diff'] <= 255, "max_pixel_diff must be 0-255" + + if 'max_pixel_diff_percent' in tolerance: + assert isinstance(tolerance['max_pixel_diff_percent'], (int, float)), "max_pixel_diff_percent must be numeric" + assert 0 <= tolerance['max_pixel_diff_percent'] <= 100, "max_pixel_diff_percent must be 0-100%" + + if 'allowed_diff_pixels' in tolerance: + assert isinstance(tolerance['allowed_diff_pixels'], (int, float)), "allowed_diff_pixels must be numeric" + assert 0 <= tolerance['allowed_diff_pixels'] <= 100, "allowed_diff_pixels must be 0-100%" + class TestConfig(RenderingConfig): def __init__(self, data, existing_models, presets): RenderingConfig.__init__(self, data) @@ -72,6 +120,7 @@ class TestConfig(RenderingConfig): apply_presets = data.get('apply_presets') rendering = {} preset_models = [] + preset_tolerance = None if apply_presets: given_presets = {p.name: p for p in presets} assert all((name in given_presets) for name in apply_presets),\ @@ -79,9 +128,12 @@ class TestConfig(RenderingConfig): # Note that this needs to applied in order. Models will be overwritten. # Properties will be "added" in order. + # Tolerance is inherited from the LAST preset that has one defined for preset in apply_presets: rendering.update(given_presets[preset].rendering) preset_models = given_presets[preset].models + if given_presets[preset].tolerance: + preset_tolerance = given_presets[preset].tolerance assert 'rendering' in data rendering.update(data['rendering']) @@ -94,6 +146,22 @@ class TestConfig(RenderingConfig): assert all(m in existing_models for m in models) self.models = set(models + self.models) + # Parse tolerance configuration - test-level tolerance overrides preset tolerance + tolerance = data.get('tolerance') + if tolerance: + assert _is_dict(tolerance) + self._validate_tolerance(tolerance) + self.tolerance = tolerance + else: + # Use tolerance inherited from presets + self.tolerance = preset_tolerance + + def _validate_tolerance(self, tolerance): + """Use the same validation logic as PresetConfig.""" + # Create a temporary PresetConfig instance to reuse validation logic + temp_preset = PresetConfig({'name': 'temp', 'rendering': {}}, {}) + return temp_preset._validate_tolerance(tolerance) + def to_filament_format(self): json_out = { 'name': self.name, diff --git a/test/renderdiff/tests/presubmit.json b/test/renderdiff/tests/presubmit.json index a46d0b9811..3add64b444 100644 --- a/test/renderdiff/tests/presubmit.json +++ b/test/renderdiff/tests/presubmit.json @@ -10,6 +10,10 @@ "viewer.cameraFocusDistance": 0, "view.postProcessingEnabled": true, "view.dithering": "NONE" + }, + "tolerance": { // [optional] Simple tolerance - single criteria + "max_pixel_diff": 5, // Max absolute difference per channel (0-255) + "allowed_diff_pixels": 0.1 // Percentage of pixels allowed to exceed (0-100%) } }, { diff --git a/test/renderdiff/tests/sample.json b/test/renderdiff/tests/sample.json index eb689d8102..01b7651991 100644 --- a/test/renderdiff/tests/sample.json +++ b/test/renderdiff/tests/sample.json @@ -25,6 +25,19 @@ // 'presets' field in the top-level struct. "model": [], // [optional] List of models used in this test. The list is // *appended* onto the lists provided by the presets. + "tolerance": { // [optional] Image comparison tolerance specification + "operator": "OR", // [optional] How to combine criteria: "AND" or "OR" (default: "AND") + "criteria": [ + { + "max_pixel_diff": 3, // [optional] Max absolute difference per channel (0-255) + "allowed_diff_pixels": 1.0 // [optional] Percentage of pixels allowed to exceed (0-100%) + }, + { + "max_pixel_diff_percent": 2.0, // [optional] Max difference as percentage (0-100%) + "allowed_diff_pixels": 0.5 // [optional] Percentage of pixels allowed to exceed (0-100%) + } + ] + }, "rendering": { // [required] See description in 'presets' "view.bloom.enabled": true, "view.bloom.lensFlare": true