Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

android: [render-validation] add more test result details (#9812)

Browse Source 
Add the following information:
 - Android build fingerprint, version
 - GPU driver name, info, vendor name
 - Time elapsed for test
 - Rendered images (as oppose to diff image)

filament-utils:
 - Add DeviceUtils to hook into Platform methods for reading out
   strings about gpu vendor, driver.

Fix "tolerance" in test definition
This commit is contained in:
Powei Feng
2026-03-19 16:03:00 -07:00
committed by GitHub
parent 81961f6d49
commit d65589bf77
7 changed files with 165 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
android/filament-utils-android/CMakeLists.txt
View File

@@ -61,6 +61,7 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,max-page-size
add_library(filament-utils-jni SHARED
src/main/cpp/AutomationEngine.cpp
src/main/cpp/Bookmark.cpp
src/main/cpp/DeviceUtils.cpp
src/main/cpp/HDRLoader.cpp
src/main/cpp/IBLPrefilterContext.cpp
src/main/cpp/Utils.cpp

85
android/filament-utils-android/src/main/cpp/DeviceUtils.cpp Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright (C) 2026 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <jni.h>
#include <filament/Engine.h>
#include <backend/Platform.h>
#include <utils/CString.h>
#include <algorithm>
#include <array>
using namespace filament;
namespace {
constexpr std::array<backend::Platform::DeviceInfoType, 3> VULKAN_INFO = {
backend::Platform::DeviceInfoType::VULKAN_DEVICE_NAME,
backend::Platform::DeviceInfoType::VULKAN_DRIVER_NAME,
backend::Platform::DeviceInfoType::VULKAN_DRIVER_INFO,
};
constexpr std::array<backend::Platform::DeviceInfoType, 3> GL_INFO = {
backend::Platform::DeviceInfoType::OPENGL_VENDOR,
backend::Platform::DeviceInfoType::OPENGL_RENDERER,
backend::Platform::DeviceInfoType::OPENGL_VERSION,
};
} // namespace
extern "C" JNIEXPORT jstring JNICALL
Java_com_google_android_filament_utils_DeviceUtils_nGetGpuDriverInfo(JNIEnv* env, jclass,
jlong nativeEngine) {
auto emptyStr = [env]() { return env->NewStringUTF(""); };
Engine* engine = (Engine*) nativeEngine;
if (!engine) {
return emptyStr();
}
backend::Platform* platform = engine->getPlatform();
if (!platform) {
return emptyStr();
}
std::array<backend::Platform::DeviceInfoType, 3> infoTypes;
switch (engine->getBackend()) {
case backend::Backend::VULKAN:
infoTypes = VULKAN_INFO;
break;
case backend::Backend::OPENGL:
infoTypes = GL_INFO;
break;
default:
return emptyStr();
}
backend::Driver* driver = const_cast<backend::Driver*>(engine->getDriver());
utils::CString fullInfo;
std::for_each(infoTypes.begin(), infoTypes.end(),
[&](backend::Platform::DeviceInfoType infoType) {
utils::CString const newInfo = platform->getDeviceInfo(infoType, driver);
if (!newInfo.empty()) {
if (!fullInfo.empty()) {
fullInfo += " | ";
}
fullInfo += newInfo.c_str();
}
});
return env->NewStringUTF(fullInfo.c_str());
}

26
android/filament-utils-android/src/main/java/com/google/android/filament/utils/DeviceUtils.java Normal file
View File

@@ -0,0 +1,26 @@
/*
* Copyright (C) 2026 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.filament.utils;
import com.google.android.filament.Engine;
public class DeviceUtils {
public static String getGpuDriverInfo(Engine engine) {
return nGetGpuDriverInfo(engine.getNativeObject());
}
private static native String nGetGpuDriverInfo(long nativeEngine);
}

8
android/samples/sample-render-validation/src/main/assets/default_test.json
View File

@@ -30,11 +30,11 @@
"view": {
"postProcessingEnabled": true,
"dithering": "NONE"
},
"tolerance": {
"maxAbsDiff": 0.1,
"maxFailingPixelsFraction": 0.0
}
},
"tolerance": {
"maxAbsDiff": 0.1,
"maxFailingPixelsFraction": 0.0
}
},
{

18
android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/MainActivity.kt
View File

@@ -368,6 +368,18 @@ class MainActivity : Activity(), ValidationRunner.Callback {
}
}
private fun createResultManager(outputDir: File): ValidationResultManager {
val gpuDriverInfo = com.google.android.filament.utils.DeviceUtils.getGpuDriverInfo(modelViewer.engine)
return ValidationResultManager(
outputDir = outputDir,
gpuDriverInfo = gpuDriverInfo,
deviceName = android.os.Build.MODEL,
deviceCodeName = android.os.Build.DEVICE,
androidVersion = android.os.Build.VERSION.RELEASE,
androidBuildNumber = android.os.Build.DISPLAY
)
}
private fun startValidation(input: ValidationInputManager.ValidationInput) {
try {
resultsContainer.removeAllViews()
@@ -378,7 +390,7 @@ class MainActivity : Activity(), ValidationRunner.Callback {
testResultsHeader.text = "${input.config.name}"
resultManager = ValidationResultManager(input.outputDir)
resultManager = createResultManager(input.outputDir)
validationRunner = ValidationRunner(this, modelViewer, input.config, resultManager!!)
validationRunner?.callback = this
@@ -534,7 +546,7 @@ class MainActivity : Activity(), ValidationRunner.Callback {
private fun exportTestBundleAction() {
currentInput?.let { input ->
val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date())
val rm = resultManager ?: ValidationResultManager(input.outputDir)
val rm = resultManager ?: createResultManager(input.outputDir)
val zip = rm.exportTestBundle(input.config, timestamp)
if (zip != null) {
val msg = "Exported Bundle: ${zip.name}"
@@ -550,7 +562,7 @@ class MainActivity : Activity(), ValidationRunner.Callback {
private fun exportTestResultsAction() {
currentInput?.let { input ->
val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date())
val rm = resultManager ?: ValidationResultManager(input.outputDir)
val rm = resultManager ?: createResultManager(input.outputDir)
val zip = rm.exportTestResults(input.sourceZip, timestamp)
if (zip != null) {
val msg = "Exported Results: ${zip.name}"

37
android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationResultManager.kt
View File

@@ -31,7 +31,14 @@ data class ValidationResult(
val diffMetric: Float = 0f
)
class ValidationResultManager(private val outputDir: File) {
class ValidationResultManager(
private val outputDir: File,
private val gpuDriverInfo: String,
private val deviceName: String,
private val deviceCodeName: String,
private val androidVersion: String,
private val androidBuildNumber: String
) {
companion object {
private const val TAG = "ValidationResultManager"
@@ -64,9 +71,9 @@ class ValidationResultManager(private val outputDir: File) {
return outputDir
}
fun finalizeResults(): File? {
fun finalizeResults(totalTimeMs: Long): File? {
// Write results JSON
writeResultsJson()
writeResultsJson(totalTimeMs)
return null
}
@@ -103,10 +110,10 @@ class ValidationResultManager(private val outputDir: File) {
zos.closeEntry()
}
// 3. Add diff images (any file ending in _diff.png in outputDir)
outputDir.listFiles { _, name -> name.endsWith("_diff.png") }?.forEach { diffFile ->
zos.putNextEntry(ZipEntry(diffFile.name))
diffFile.inputStream().use { it.copyTo(zos) }
// 3. Add images (only rendered images, exclude diffs)
outputDir.listFiles { _, name -> name.endsWith(".png") && !name.endsWith("_diff.png") }?.forEach { imgFile ->
zos.putNextEntry(ZipEntry(imgFile.name))
imgFile.inputStream().use { it.copyTo(zos) }
zos.closeEntry()
}
}
@@ -266,7 +273,18 @@ class ValidationResultManager(private val outputDir: File) {
}
}
private fun writeResultsJson() {
private fun writeResultsJson(totalTimeMs: Long) {
val rootObject = JSONObject()
val metadataObject = JSONObject()
metadataObject.put("gpu_driver_info", gpuDriverInfo ?: "")
metadataObject.put("total_time_ms", totalTimeMs)
metadataObject.put("device_name", deviceName ?: "")
metadataObject.put("device_code_name", deviceCodeName ?: "")
metadataObject.put("android_version", androidVersion ?: "")
metadataObject.put("android_build_number", androidBuildNumber ?: "")
rootObject.put("metadata", metadataObject)
val jsonArray = JSONArray()
for (result in results) {
val jsonObject = JSONObject()
@@ -275,11 +293,12 @@ class ValidationResultManager(private val outputDir: File) {
jsonObject.put("diff_metric", result.diffMetric)
jsonArray.put(jsonObject)
}
rootObject.put("results", jsonArray)
val jsonFile = File(outputDir, "results.json")
try {
FileOutputStream(jsonFile).use { out ->
out.write(jsonArray.toString(4).toByteArray())
out.write(rootObject.toString(4).toByteArray())
}
} catch (e: Exception) {
Log.e(TAG, "Failed to write results.json", e)

7
android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationRunner.kt
View File

@@ -42,6 +42,7 @@ class ValidationRunner(
private var currentModelName: String? = null
private var frameCounter = 0
private var suiteStartTime: Long = 0
enum class State {
IDLE,
@@ -65,6 +66,7 @@ class ValidationRunner(
callback?.onAllTestsFinished()
return
}
suiteStartTime = System.currentTimeMillis()
currentTestIndex = 0
currentModelIndex = 0
startTest(config.tests[0])
@@ -356,7 +358,10 @@ class ValidationRunner(
startTest(config.tests[currentTestIndex])
} else {
currentState = State.IDLE
resultManager.finalizeResults()
val totalTimeMs = System.currentTimeMillis() - suiteStartTime
resultManager.finalizeResults(totalTimeMs)
callback?.onAllTestsFinished()
}
}