From d65589bf77c8119acf96dbc58ffd2fc704ee19eb Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Thu, 19 Mar 2026 16:03:00 -0700 Subject: [PATCH] android: [render-validation] add more test result details (#9812) Add the following information: - Android build fingerprint, version - GPU driver name, info, vendor name - Time elapsed for test - Rendered images (as oppose to diff image) filament-utils: - Add DeviceUtils to hook into Platform methods for reading out strings about gpu vendor, driver. Fix "tolerance" in test definition --- android/filament-utils-android/CMakeLists.txt | 1 + .../src/main/cpp/DeviceUtils.cpp | 85 +++++++++++++++++++ .../android/filament/utils/DeviceUtils.java | 26 ++++++ .../src/main/assets/default_test.json | 8 +- .../filament/validation/MainActivity.kt | 18 +++- .../validation/ValidationResultManager.kt | 37 ++++++-- .../filament/validation/ValidationRunner.kt | 7 +- 7 files changed, 165 insertions(+), 17 deletions(-) create mode 100644 android/filament-utils-android/src/main/cpp/DeviceUtils.cpp create mode 100644 android/filament-utils-android/src/main/java/com/google/android/filament/utils/DeviceUtils.java diff --git a/android/filament-utils-android/CMakeLists.txt b/android/filament-utils-android/CMakeLists.txt index 3e00ee0aaa..4633ba7448 100644 --- a/android/filament-utils-android/CMakeLists.txt +++ b/android/filament-utils-android/CMakeLists.txt @@ -61,6 +61,7 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,max-page-size add_library(filament-utils-jni SHARED src/main/cpp/AutomationEngine.cpp src/main/cpp/Bookmark.cpp + src/main/cpp/DeviceUtils.cpp src/main/cpp/HDRLoader.cpp src/main/cpp/IBLPrefilterContext.cpp src/main/cpp/Utils.cpp diff --git a/android/filament-utils-android/src/main/cpp/DeviceUtils.cpp b/android/filament-utils-android/src/main/cpp/DeviceUtils.cpp new file mode 100644 index 0000000000..1b98e61c8b --- /dev/null +++ b/android/filament-utils-android/src/main/cpp/DeviceUtils.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include + +#include + +#include +#include + +using namespace filament; + +namespace { + +constexpr std::array VULKAN_INFO = { + backend::Platform::DeviceInfoType::VULKAN_DEVICE_NAME, + backend::Platform::DeviceInfoType::VULKAN_DRIVER_NAME, + backend::Platform::DeviceInfoType::VULKAN_DRIVER_INFO, +}; + +constexpr std::array GL_INFO = { + backend::Platform::DeviceInfoType::OPENGL_VENDOR, + backend::Platform::DeviceInfoType::OPENGL_RENDERER, + backend::Platform::DeviceInfoType::OPENGL_VERSION, +}; + +} // namespace + +extern "C" JNIEXPORT jstring JNICALL +Java_com_google_android_filament_utils_DeviceUtils_nGetGpuDriverInfo(JNIEnv* env, jclass, + jlong nativeEngine) { + auto emptyStr = [env]() { return env->NewStringUTF(""); }; + Engine* engine = (Engine*) nativeEngine; + if (!engine) { + return emptyStr(); + } + + backend::Platform* platform = engine->getPlatform(); + if (!platform) { + return emptyStr(); + } + + std::array infoTypes; + switch (engine->getBackend()) { + case backend::Backend::VULKAN: + infoTypes = VULKAN_INFO; + break; + case backend::Backend::OPENGL: + infoTypes = GL_INFO; + break; + default: + return emptyStr(); + } + + backend::Driver* driver = const_cast(engine->getDriver()); + utils::CString fullInfo; + std::for_each(infoTypes.begin(), infoTypes.end(), + [&](backend::Platform::DeviceInfoType infoType) { + utils::CString const newInfo = platform->getDeviceInfo(infoType, driver); + if (!newInfo.empty()) { + if (!fullInfo.empty()) { + fullInfo += " | "; + } + fullInfo += newInfo.c_str(); + } + }); + return env->NewStringUTF(fullInfo.c_str()); +} diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/DeviceUtils.java b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/DeviceUtils.java new file mode 100644 index 0000000000..790e8de462 --- /dev/null +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/DeviceUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.filament.utils; + +import com.google.android.filament.Engine; + +public class DeviceUtils { + public static String getGpuDriverInfo(Engine engine) { + return nGetGpuDriverInfo(engine.getNativeObject()); + } + private static native String nGetGpuDriverInfo(long nativeEngine); +} diff --git a/android/samples/sample-render-validation/src/main/assets/default_test.json b/android/samples/sample-render-validation/src/main/assets/default_test.json index dc4bf1ed4c..11e318422d 100644 --- a/android/samples/sample-render-validation/src/main/assets/default_test.json +++ b/android/samples/sample-render-validation/src/main/assets/default_test.json @@ -30,11 +30,11 @@ "view": { "postProcessingEnabled": true, "dithering": "NONE" - }, - "tolerance": { - "maxAbsDiff": 0.1, - "maxFailingPixelsFraction": 0.0 } + }, + "tolerance": { + "maxAbsDiff": 0.1, + "maxFailingPixelsFraction": 0.0 } }, { diff --git a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/MainActivity.kt b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/MainActivity.kt index f0fecf45a8..44c7d595c1 100644 --- a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/MainActivity.kt +++ b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/MainActivity.kt @@ -368,6 +368,18 @@ class MainActivity : Activity(), ValidationRunner.Callback { } } + private fun createResultManager(outputDir: File): ValidationResultManager { + val gpuDriverInfo = com.google.android.filament.utils.DeviceUtils.getGpuDriverInfo(modelViewer.engine) + return ValidationResultManager( + outputDir = outputDir, + gpuDriverInfo = gpuDriverInfo, + deviceName = android.os.Build.MODEL, + deviceCodeName = android.os.Build.DEVICE, + androidVersion = android.os.Build.VERSION.RELEASE, + androidBuildNumber = android.os.Build.DISPLAY + ) + } + private fun startValidation(input: ValidationInputManager.ValidationInput) { try { resultsContainer.removeAllViews() @@ -378,7 +390,7 @@ class MainActivity : Activity(), ValidationRunner.Callback { testResultsHeader.text = "${input.config.name}" - resultManager = ValidationResultManager(input.outputDir) + resultManager = createResultManager(input.outputDir) validationRunner = ValidationRunner(this, modelViewer, input.config, resultManager!!) validationRunner?.callback = this @@ -534,7 +546,7 @@ class MainActivity : Activity(), ValidationRunner.Callback { private fun exportTestBundleAction() { currentInput?.let { input -> val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date()) - val rm = resultManager ?: ValidationResultManager(input.outputDir) + val rm = resultManager ?: createResultManager(input.outputDir) val zip = rm.exportTestBundle(input.config, timestamp) if (zip != null) { val msg = "Exported Bundle: ${zip.name}" @@ -550,7 +562,7 @@ class MainActivity : Activity(), ValidationRunner.Callback { private fun exportTestResultsAction() { currentInput?.let { input -> val timestamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date()) - val rm = resultManager ?: ValidationResultManager(input.outputDir) + val rm = resultManager ?: createResultManager(input.outputDir) val zip = rm.exportTestResults(input.sourceZip, timestamp) if (zip != null) { val msg = "Exported Results: ${zip.name}" diff --git a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationResultManager.kt b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationResultManager.kt index 467f0a874d..2b808f9294 100644 --- a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationResultManager.kt +++ b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationResultManager.kt @@ -31,7 +31,14 @@ data class ValidationResult( val diffMetric: Float = 0f ) -class ValidationResultManager(private val outputDir: File) { +class ValidationResultManager( + private val outputDir: File, + private val gpuDriverInfo: String, + private val deviceName: String, + private val deviceCodeName: String, + private val androidVersion: String, + private val androidBuildNumber: String +) { companion object { private const val TAG = "ValidationResultManager" @@ -64,9 +71,9 @@ class ValidationResultManager(private val outputDir: File) { return outputDir } - fun finalizeResults(): File? { + fun finalizeResults(totalTimeMs: Long): File? { // Write results JSON - writeResultsJson() + writeResultsJson(totalTimeMs) return null } @@ -103,10 +110,10 @@ class ValidationResultManager(private val outputDir: File) { zos.closeEntry() } - // 3. Add diff images (any file ending in _diff.png in outputDir) - outputDir.listFiles { _, name -> name.endsWith("_diff.png") }?.forEach { diffFile -> - zos.putNextEntry(ZipEntry(diffFile.name)) - diffFile.inputStream().use { it.copyTo(zos) } + // 3. Add images (only rendered images, exclude diffs) + outputDir.listFiles { _, name -> name.endsWith(".png") && !name.endsWith("_diff.png") }?.forEach { imgFile -> + zos.putNextEntry(ZipEntry(imgFile.name)) + imgFile.inputStream().use { it.copyTo(zos) } zos.closeEntry() } } @@ -266,7 +273,18 @@ class ValidationResultManager(private val outputDir: File) { } } - private fun writeResultsJson() { + private fun writeResultsJson(totalTimeMs: Long) { + val rootObject = JSONObject() + + val metadataObject = JSONObject() + metadataObject.put("gpu_driver_info", gpuDriverInfo ?: "") + metadataObject.put("total_time_ms", totalTimeMs) + metadataObject.put("device_name", deviceName ?: "") + metadataObject.put("device_code_name", deviceCodeName ?: "") + metadataObject.put("android_version", androidVersion ?: "") + metadataObject.put("android_build_number", androidBuildNumber ?: "") + rootObject.put("metadata", metadataObject) + val jsonArray = JSONArray() for (result in results) { val jsonObject = JSONObject() @@ -275,11 +293,12 @@ class ValidationResultManager(private val outputDir: File) { jsonObject.put("diff_metric", result.diffMetric) jsonArray.put(jsonObject) } + rootObject.put("results", jsonArray) val jsonFile = File(outputDir, "results.json") try { FileOutputStream(jsonFile).use { out -> - out.write(jsonArray.toString(4).toByteArray()) + out.write(rootObject.toString(4).toByteArray()) } } catch (e: Exception) { Log.e(TAG, "Failed to write results.json", e) diff --git a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationRunner.kt b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationRunner.kt index e75b4a4fea..53fa4baf68 100644 --- a/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationRunner.kt +++ b/android/samples/sample-render-validation/src/main/java/com/google/android/filament/validation/ValidationRunner.kt @@ -42,6 +42,7 @@ class ValidationRunner( private var currentModelName: String? = null private var frameCounter = 0 + private var suiteStartTime: Long = 0 enum class State { IDLE, @@ -65,6 +66,7 @@ class ValidationRunner( callback?.onAllTestsFinished() return } + suiteStartTime = System.currentTimeMillis() currentTestIndex = 0 currentModelIndex = 0 startTest(config.tests[0]) @@ -356,7 +358,10 @@ class ValidationRunner( startTest(config.tests[currentTestIndex]) } else { currentState = State.IDLE - resultManager.finalizeResults() + + val totalTimeMs = System.currentTimeMillis() - suiteStartTime + + resultManager.finalizeResults(totalTimeMs) callback?.onAllTestsFinished() } }