vk: fix VulkanPlatform CString usage (#9266) (#9260)

Since #9259, CString in VulkanPlatform fall into the literal
constructor path. But we really want the null-terminated
(char const*) path.  So we cast the strings to (char const*)
to enforce null-teriminated behavior.

RDIFF_BRANCH=pf/renderdiff-add-tolerance
This commit is contained in:
Powei Feng
2025-09-26 16:42:56 -07:00
committed by GitHub
parent 33a4ab3d88
commit d8e8aafad7
7 changed files with 391 additions and 27 deletions

View File

@@ -8,8 +8,81 @@ import fnmatch
from utils import execute, ArgParseImpl, important_print, mkdir_p
from image_diff import same_image, output_image_diff
from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING, GOLDEN_MISSING
import test_config
def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None):
def _get_tolerance_for_test_case(test_case_name, test_config_obj):
if not test_config_obj:
return None
# Extract test name from test case (remove backend and model)
# Format: TestName.backend.model -> TestName
test_name = test_case_name.split('.')[0]
for test in test_config_obj.tests:
if test.name == test_name:
return test.tolerance
return None
def _format_tolerance_summary(stats):
"""
Create human-readable summary of tolerance statistics.
Args:
stats: Statistics dictionary from tolerance evaluation
Returns:
str: Formatted summary string
"""
if 'error' in stats:
return f"Error: {stats['error']}"
if 'operator' in stats:
# Nested criteria with operator
operator = stats['operator']
criteria_count = len(stats['criteria_results'])
passed_count = sum(1 for c in stats['criteria_results'] if c.get('passed', False))
summary = f"{operator} of {criteria_count} criteria: {passed_count} passed, {criteria_count - passed_count} failed"
# Add details for each criteria
details = []
for i, criteria_stats in enumerate(stats['criteria_results']):
details.append(f" Criteria {i+1}: {_format_tolerance_summary(criteria_stats)}")
return summary + "\n" + "\n".join(details)
else:
# Single criteria
total_pixels = stats.get('total_pixels', 0)
failing_pixels = stats.get('failing_pixels', 0)
failing_percentage = stats.get('failing_percentage', 0.0)
allowed_percentage = stats.get('allowed_percentage', 0.0)
max_abs_diff = stats.get('max_abs_diff', 0)
mean_abs_diff = stats.get('mean_abs_diff', 0)
max_diff_per_channel = stats.get('max_diff_per_channel', [])
criteria = stats.get('criteria', {})
criteria_desc = []
if 'max_pixel_diff' in criteria:
criteria_desc.append(f"max_pixel_diff: {criteria['max_pixel_diff']}")
if 'max_pixel_diff_percent' in criteria:
criteria_desc.append(f"max_pixel_diff_percent: {criteria['max_pixel_diff_percent']}%")
if 'allowed_diff_pixels' in criteria:
criteria_desc.append(f"allowed_diff_pixels: {criteria['allowed_diff_pixels']}%")
summary_lines = [
f"Tolerance: {', '.join(criteria_desc)}",
f"Pixels: {failing_pixels:,} / {total_pixels:,} ({failing_percentage:.2f}%) exceed tolerance",
f"Allowed: {allowed_percentage:.2f}% - {'PASS' if stats.get('passed', False) else 'FAIL'}",
f"Max difference: {max_abs_diff} (mean: {mean_abs_diff:.1f})"
]
if len(max_diff_per_channel) > 1:
channel_info = ", ".join(f"Ch{i}: {diff}" for i, diff in enumerate(max_diff_per_channel))
summary_lines.append(f"Per-channel max: {channel_info}")
return "\n".join(summary_lines)
def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None, test_config_path=None):
def test_name(p):
return p.replace('.tif', '')
@@ -20,6 +93,14 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None):
for f in all_files)
all_results = []
# Parse test configuration if provided
test_config_obj = None
if test_config_path and os.path.exists(test_config_path):
try:
test_config_obj = test_config.parse_from_path(test_config_path)
except Exception as e:
important_print(f"Warning: Could not parse test config {test_config_path}: {e}")
def single_test(src_dir, dest_dir, src_fname):
src_fname = os.path.abspath(src_fname)
test_case = src_fname.replace(f'{src_dir}/', '')
@@ -27,17 +108,41 @@ def _compare_goldens(base_dir, comparison_dir, out_dir=None, test_filter=None):
result = {
'name': test_case,
}
if not os.path.exists(dest_fname):
result['result'] = RESULT_MISSING
elif not same_image(src_fname, dest_fname):
result['result'] = RESULT_FAILED
if output_test_dir:
# just the file name
diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
output_image_diff(src_fname, dest_fname, os.path.join(output_test_dir, diff_fname))
result['diff'] = diff_fname
else:
result['result'] = RESULT_OK
# Get tolerance configuration for this test case
tolerance = _get_tolerance_for_test_case(test_case.replace('.tif', ''), test_config_obj)
# Compare images and get detailed statistics
comparison_result, stats = same_image(src_fname, dest_fname, tolerance)
if not comparison_result:
result['result'] = RESULT_FAILED
if output_test_dir:
# just the file name
diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
output_image_diff(src_fname, dest_fname, os.path.join(output_test_dir, diff_fname))
result['diff'] = diff_fname
else:
result['result'] = RESULT_OK
# Add detailed tolerance information to result
if tolerance:
result['tolerance_used'] = True
result['tolerance_config'] = tolerance
if stats:
result['tolerance_stats'] = stats
# Add human-readable summary
result['tolerance_summary'] = _format_tolerance_summary(stats)
elif stats is None and comparison_result:
result['comparison_type'] = 'exact_match'
elif stats and 'error' in stats:
result['error'] = stats['error']
if 'details' in stats:
result['error_details'] = stats['details']
return result
for test_dir in test_dirs:
@@ -87,6 +192,7 @@ if __name__ == '__main__':
parser.add_argument('--dest', help='Directory of the comparison of the diff.')
parser.add_argument('--out', help='Directory of output for the result of the diff.')
parser.add_argument('--test_filter', help='Filter for the tests to run')
parser.add_argument('--test', help='Path to test configuration JSON file for tolerance settings.')
args, _ = parser.parse_known_args(sys.argv[1:])
@@ -96,11 +202,42 @@ if __name__ == '__main__':
dest = os.path.join(os.getcwd(), './out/renderdiff')
assert os.path.exists(dest), f"Destination folder={dest} does not exist."
results = _compare_goldens(args.src, dest, out_dir=args.out, test_filter=args.test_filter)
results = _compare_goldens(args.src, dest, out_dir=args.out,
test_filter=args.test_filter, test_config_path=args.test)
failed = [f" {k['name']} ({k['result']})" for k in results if k['result'] != RESULT_OK]
success_count = len(results) - len(failed)
important_print(f'Successfully compared {success_count} / {len(results)} images' +
('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))
# Categorize results
failed = [k for k in results if k['result'] != RESULT_OK]
passed = [k for k in results if k['result'] == RESULT_OK]
tolerance_used_count = len([k for k in results if k.get('tolerance_used', False)])
# Create detailed failure report
failed_details = []
for k in failed:
failure_line = f" {k['name']} ({k['result']})"
if 'tolerance_summary' in k:
failure_line += f"\n {k['tolerance_summary'].replace(chr(10), chr(10) + ' ')}"
failed_details.append(failure_line)
# Create success report with tolerance details
tolerance_used_details = []
for k in passed:
if k.get('tolerance_used', False) and 'tolerance_summary' in k:
tolerance_used_details.append(f" {k['name']}: {k['tolerance_summary'].split(chr(10))[0]}")
# Main summary
success_count = len(passed)
important_print(f'Successfully compared {success_count} / {len(results)} images')
if tolerance_used_details:
pstr = 'Tolerance-based passes:'
for detail in tolerance_used_details:
pstr += '\n' + detail
important_print(pstr)
if failed_details:
pstr = 'Failed:'
for detail in failed_details:
pstr = '\n' + detail
important_print(pstr)
if len(failed) > 0:
exit(1)

View File

@@ -15,7 +15,144 @@
import tifffile
import numpy as np
def same_image(tiff_file_a, tiff_file_b):
def evaluate_tolerance_criteria(img1_data, img2_data, tolerance_spec):
"""
Recursively evaluate tolerance criteria with AND/OR logic.
Args:
img1_data: First image array
img2_data: Second image array
tolerance_spec: Dictionary containing tolerance criteria with structure:
{
"operator": "AND" | "OR", # How to combine criteria results
"criteria": [...] # List of criteria or nested tolerance specs
}
OR for leaf criteria:
{
"max_pixel_diff": int, # Max absolute difference per channel (0-255)
"max_pixel_diff_percent": float, # Max difference as percentage (0-100%)
"allowed_diff_pixels": float # Percentage of pixels allowed to exceed (0-100%)
}
Returns:
tuple: (bool: pass/fail, dict: detailed statistics)
"""
if 'criteria' not in tolerance_spec:
# Leaf criteria - evaluate single condition
return evaluate_single_criteria(img1_data, img2_data, tolerance_spec)
operator = tolerance_spec.get('operator', 'AND').upper()
criteria_list = tolerance_spec['criteria']
results_and_stats = [evaluate_tolerance_criteria(img1_data, img2_data, criteria)
for criteria in criteria_list]
results = [r[0] for r in results_and_stats] # Extract pass/fail results
all_stats = [r[1] for r in results_and_stats] # Extract statistics
if operator == 'AND':
final_result = all(results)
elif operator == 'OR':
final_result = any(results)
else:
raise ValueError(f"Unknown operator: {operator}")
# Combine statistics
combined_stats = {
'operator': operator,
'criteria_results': all_stats,
'passed': bool(final_result)
}
return final_result, combined_stats
def evaluate_single_criteria(img1_data, img2_data, criteria):
"""
Evaluate a single tolerance criteria.
Args:
img1_data: First image array
img2_data: Second image array
criteria: Dictionary with tolerance parameters:
- max_pixel_diff: Maximum absolute difference per channel (0-255 range)
- max_pixel_diff_percent: Maximum difference as percentage (0-100%)
- allowed_diff_pixels: Percentage of pixels allowed to exceed tolerance (0-100%)
Returns:
tuple: (bool: pass/fail, dict: detailed statistics)
"""
diff_abs = np.abs(img1_data.astype(np.int16) - img2_data.astype(np.int16))
max_diff = criteria.get('max_pixel_diff', float('inf'))
max_diff_percent = criteria.get('max_pixel_diff_percent', float('inf'))
allowed_diff_pixels = criteria.get('allowed_diff_pixels', 0.0)
# Calculate which pixels exceed absolute threshold
exceeds_abs = diff_abs > max_diff if max_diff < float('inf') else np.zeros_like(diff_abs, dtype=bool)
# Calculate which pixels exceed percentage threshold
if max_diff_percent < float('inf'):
max_val = np.maximum(img1_data, img2_data)
# Avoid division by zero
diff_percent = np.divide(diff_abs * 100.0, np.maximum(max_val, 1),
out=np.zeros_like(diff_abs, dtype=np.float32),
where=max_val!=0)
exceeds_percent = diff_percent > max_diff_percent
else:
exceeds_percent = np.zeros_like(diff_abs, dtype=bool)
# A pixel fails if it exceeds either threshold (OR logic at pixel level)
exceeds_tolerance = exceeds_abs | exceeds_percent
# Check per-pixel: pixel fails if ANY channel exceeds tolerance
exceeds_per_pixel = np.any(exceeds_tolerance, axis=-1) if len(exceeds_tolerance.shape) > 2 else exceeds_tolerance
# Calculate detailed statistics
total_pixels = exceeds_per_pixel.size
failing_pixels = np.sum(exceeds_per_pixel)
failing_percentage = (failing_pixels / total_pixels) * 100.0
# Distribution analysis
max_abs_diff = np.max(diff_abs) if diff_abs.size > 0 else 0
mean_abs_diff = np.mean(diff_abs) if diff_abs.size > 0 else 0
# Per-channel max differences
if len(diff_abs.shape) > 2:
max_diff_per_channel = [np.max(diff_abs[:, :, c]) for c in range(diff_abs.shape[2])]
else:
max_diff_per_channel = [max_abs_diff]
stats = {
'criteria': criteria,
'total_pixels': int(total_pixels),
'failing_pixels': int(failing_pixels),
'failing_percentage': float(failing_percentage),
'allowed_percentage': float(allowed_diff_pixels),
'max_abs_diff': int(max_abs_diff),
'mean_abs_diff': float(mean_abs_diff),
'max_diff_per_channel': [int(x) for x in max_diff_per_channel],
'passed': bool(failing_percentage <= allowed_diff_pixels)
}
return failing_percentage <= allowed_diff_pixels, stats
def same_image(tiff_file_a, tiff_file_b, tolerance=None):
"""
Compare two TIFF images for equality with optional tolerance.
Args:
tiff_file_a: Path to first TIFF file
tiff_file_b: Path to second TIFF file
tolerance: Optional tolerance specification dictionary. If None, performs exact comparison.
Can be either:
1. Single criteria: {"max_pixel_diff": 5, "allowed_diff_pixels": 1.0}
2. Nested criteria: {"operator": "OR", "criteria": [...]}
Returns:
tuple: (bool: pass/fail, dict: detailed statistics or None)
For exact comparison, returns (bool, None)
"""
try:
img1_data = tifffile.imread(tiff_file_a)
img2_data = tifffile.imread(tiff_file_b)
@@ -24,23 +161,25 @@ def same_image(tiff_file_a, tiff_file_b):
# are different, the images are not the same.
if img1_data.shape != img2_data.shape:
print(f"Images have different shapes: {img1_data.shape} vs {img2_data.shape}")
return False
return False, {'error': 'Shape mismatch', 'shape1': img1_data.shape, 'shape2': img2_data.shape}
# numpy.array_equal() checks if two arrays have the same shape and elements.
if np.array_equal(img1_data, img2_data):
return True
else:
return False
# If no tolerance specified, use exact comparison
if tolerance is None:
exact_match = np.array_equal(img1_data, img2_data)
return exact_match, None
# Use tolerance-based comparison
return evaluate_tolerance_criteria(img1_data, img2_data, tolerance)
except FileNotFoundError:
print(f"Error: One or both files not found ('{file_path1}', '{file_path2}').")
return False
print(f"Error: One or both files not found ('{tiff_file_a}', '{tiff_file_b}').")
return False, {'error': 'File not found'}
except tifffile.TiffFileError as e:
print(f"Error: One or both files are not valid TIFF files or could not be read. Details: {e}")
return False
return False, {'error': 'Invalid TIFF file', 'details': str(e)}
except Exception as e:
print(f"An unexpected error occurred: {e}")
return False
return False, {'error': 'Unexpected error', 'details': str(e)}
def output_image_diff(tiff_file_a, tiff_file_b, output_path):
try:

View File

@@ -28,6 +28,7 @@ BUILD_COMMON_DIR="$(pwd)/build/common"
os_name=$(uname -s)
if [[ "$os_name" == "Linux" ]]; then
MESA_LIB_DIR="${MESA_DIR}lib/x86_64-linux-gnu"
MESA_VK_ICD_PATH="${MESA_DIR}share/vulkan/icd.d/lvp_icd.x86_64.json"
elif [[ "$os_name" == "Darwin" ]]; then
MESA_LIB_DIR="${MESA_DIR}lib"
MESA_VK_ICD_PATH="${MESA_DIR}share/vulkan/icd.d/lvp_icd.aarch64.json"

View File

@@ -61,6 +61,54 @@ class PresetConfig(RenderingConfig):
_check(models)
self.models += models
# Parse tolerance configuration from preset
tolerance = data.get('tolerance')
if tolerance:
assert _is_dict(tolerance)
self._validate_tolerance(tolerance)
self.tolerance = tolerance
else:
self.tolerance = None
def _validate_tolerance(self, tolerance):
"""
Validate tolerance configuration structure.
Tolerance can be:
1. Single criteria: {"max_pixel_diff": 5, "allowed_diff_pixels": 1.0}
2. Nested criteria: {"operator": "OR", "criteria": [...]}
"""
if 'criteria' in tolerance:
# Nested structure with operator
operator = tolerance.get('operator', 'AND')
assert operator.upper() in ['AND', 'OR'], f"Invalid operator: {operator}"
criteria_list = tolerance['criteria']
assert isinstance(criteria_list, list), "criteria must be a list"
assert len(criteria_list) > 0, "criteria list cannot be empty"
# Recursively validate each criteria
for criteria in criteria_list:
self._validate_tolerance(criteria)
else:
# Leaf criteria - validate individual parameters
valid_keys = {'max_pixel_diff', 'max_pixel_diff_percent', 'allowed_diff_pixels'}
tolerance_keys = set(tolerance.keys())
invalid_keys = tolerance_keys - valid_keys
assert len(invalid_keys) == 0, f"Invalid tolerance keys: {invalid_keys}"
if 'max_pixel_diff' in tolerance:
assert isinstance(tolerance['max_pixel_diff'], (int, float)), "max_pixel_diff must be numeric"
assert 0 <= tolerance['max_pixel_diff'] <= 255, "max_pixel_diff must be 0-255"
if 'max_pixel_diff_percent' in tolerance:
assert isinstance(tolerance['max_pixel_diff_percent'], (int, float)), "max_pixel_diff_percent must be numeric"
assert 0 <= tolerance['max_pixel_diff_percent'] <= 100, "max_pixel_diff_percent must be 0-100%"
if 'allowed_diff_pixels' in tolerance:
assert isinstance(tolerance['allowed_diff_pixels'], (int, float)), "allowed_diff_pixels must be numeric"
assert 0 <= tolerance['allowed_diff_pixels'] <= 100, "allowed_diff_pixels must be 0-100%"
class TestConfig(RenderingConfig):
def __init__(self, data, existing_models, presets):
RenderingConfig.__init__(self, data)
@@ -72,6 +120,7 @@ class TestConfig(RenderingConfig):
apply_presets = data.get('apply_presets')
rendering = {}
preset_models = []
preset_tolerance = None
if apply_presets:
given_presets = {p.name: p for p in presets}
assert all((name in given_presets) for name in apply_presets),\
@@ -79,9 +128,12 @@ class TestConfig(RenderingConfig):
# Note that this needs to applied in order. Models will be overwritten.
# Properties will be "added" in order.
# Tolerance is inherited from the LAST preset that has one defined
for preset in apply_presets:
rendering.update(given_presets[preset].rendering)
preset_models = given_presets[preset].models
if given_presets[preset].tolerance:
preset_tolerance = given_presets[preset].tolerance
assert 'rendering' in data
rendering.update(data['rendering'])
@@ -94,6 +146,22 @@ class TestConfig(RenderingConfig):
assert all(m in existing_models for m in models)
self.models = set(models + self.models)
# Parse tolerance configuration - test-level tolerance overrides preset tolerance
tolerance = data.get('tolerance')
if tolerance:
assert _is_dict(tolerance)
self._validate_tolerance(tolerance)
self.tolerance = tolerance
else:
# Use tolerance inherited from presets
self.tolerance = preset_tolerance
def _validate_tolerance(self, tolerance):
"""Use the same validation logic as PresetConfig."""
# Create a temporary PresetConfig instance to reuse validation logic
temp_preset = PresetConfig({'name': 'temp', 'rendering': {}}, {})
return temp_preset._validate_tolerance(tolerance)
def to_filament_format(self):
json_out = {
'name': self.name,