Update the associated uniforms every frame for materials with attached Streams (#8493)

BUGS=[399959254]

filament/backend/include/backend/BufferObjectStreamDescriptor.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_BUFFEROBJECTSTREAMDESCRIPTOR_H
+#define TNT_FILAMENT_BACKEND_BUFFEROBJECTSTREAMDESCRIPTOR_H
+
+#include <backend/Handle.h>
+
+#include <vector>
+
+namespace filament::backend {
+
+/**
+ * The type of association between a buffer object and a stream.
+ */
+ enum class BufferObjectStreamAssociationType { TRANSFORM_MATRIX };
+
+/**
+ * A descriptor for a buffer object to stream association.
+ */
+class UTILS_PUBLIC BufferObjectStreamDescriptor {
+public:
+    //! creates an empty descriptor
+    BufferObjectStreamDescriptor() noexcept = default;
+
+    BufferObjectStreamDescriptor(const BufferObjectStreamDescriptor& rhs) = delete;
+    BufferObjectStreamDescriptor& operator=(const BufferObjectStreamDescriptor& rhs) = delete;
+
+    BufferObjectStreamDescriptor(BufferObjectStreamDescriptor&& rhs) = default;
+    BufferObjectStreamDescriptor& operator=(BufferObjectStreamDescriptor&& rhs) = default;
+
+    // --------------------------------------------------------------------------------------------
+
+    struct StreamDataDescriptor {
+        uint32_t offset;
+        Handle<HwStream> stream;
+        BufferObjectStreamAssociationType associationType;
+    };
+
+    std::vector<StreamDataDescriptor> mStreams;
+};
+
+} // namespace filament::backend
+
+#endif // TNT_FILAMENT_BACKEND_BUFFEROBJECTSTREAMDESCRIPTOR_H

filament/backend/include/private/backend/CommandStream.h

@@ -22,6 +22,7 @@
 #include "private/backend/Driver.h"
 
 #include <backend/BufferDescriptor.h>
+#include <backend/BufferObjectStreamDescriptor.h>
 #include <backend/CallbackHandler.h>
 #include <backend/DriverEnums.h>
 #include <backend/Handle.h>

filament/backend/include/private/backend/Driver.h

@@ -48,6 +48,7 @@
 namespace filament::backend {
 
 class BufferDescriptor;
+class BufferObjectStreamDescriptor;
 class CallbackHandler;
 class PixelBufferDescriptor;
 class Program;

filament/backend/include/private/backend/DriverAPI.inc

@@ -389,6 +389,10 @@ DECL_DRIVER_API_N(updateBufferObject,
         backend::BufferDescriptor&&, data,
         uint32_t, byteOffset)
 
+DECL_DRIVER_API_N(registerBufferObjectStreams,
+        backend::BufferObjectHandle, ibh,
+        backend::BufferObjectStreamDescriptor&&, streams)
+
 DECL_DRIVER_API_N(updateBufferObjectUnsynchronized,
         backend::BufferObjectHandle, ibh,
         backend::BufferDescriptor&&, data,

filament/backend/src/metal/MetalDriver.mm

@@ -1187,6 +1187,10 @@ void MetalDriver::updateIndexBuffer(Handle<HwIndexBuffer> ibh, BufferDescriptor&
     scheduleDestroy(std::move(data));
 }
 
+void MetalDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh, BufferObjectStreamDescriptor&& streams) {
+    // Noop
+}
+
 void MetalDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescriptor&& data,
         uint32_t byteOffset) {
     FILAMENT_CHECK_PRECONDITION(!isInRenderPass(mContext))

filament/backend/src/noop/NoopDriver.cpp

@@ -250,6 +250,8 @@ void NoopDriver::updateBufferObject(Handle<HwBufferObject> ibh, BufferDescriptor
     scheduleDestroy(std::move(p));
 }
 
+void NoopDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh, BufferObjectStreamDescriptor&& streams) { }
+
 void NoopDriver::updateBufferObjectUnsynchronized(Handle<HwBufferObject> ibh, BufferDescriptor&& p,
         uint32_t byteOffset) {
     scheduleDestroy(std::move(p));

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -38,6 +38,7 @@
 #include <backend/Platform.h>
 #include <backend/Program.h>
 #include <backend/TargetBufferInfo.h>
+#include <backend/BufferObjectStreamDescriptor.h>
 
 #include "private/backend/CommandStream.h"
 #include "private/backend/Dispatcher.h"
@@ -2551,6 +2552,38 @@ void OpenGLDriver::updateIndexBuffer(
     CHECK_GL_ERROR(utils::slog.e)
 }
 
+void OpenGLDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh, BufferObjectStreamDescriptor&& streams) {
+    DEBUG_MARKER()
+
+    GLBufferObject* bo = handle_cast<GLBufferObject*>(boh);
+    mStreamUniformDescriptors[bo->gl.id] = std::move(streams);
+}
+
+
+// specialization for mat3f (which has a different alignment, see std140 layout rules)
+static void copyMat3f(void* addr, size_t const offset, const mat3f& v) noexcept {
+    struct mat43 {
+        float v[3][4];
+    };
+
+    addr = static_cast<char*>(addr) + offset;
+    mat43& temp = *static_cast<mat43*>(addr);
+
+    temp.v[0][0] = v[0][0];
+    temp.v[0][1] = v[0][1];
+    temp.v[0][2] = v[0][2];
+
+    temp.v[1][0] = v[1][0];
+    temp.v[1][1] = v[1][1];
+    temp.v[1][2] = v[1][2];
+
+    temp.v[2][0] = v[2][0];
+    temp.v[2][1] = v[2][1];
+    temp.v[2][2] = v[2][2];
+
+    // don't store anything in temp.v[][3] because there could be uniforms packed there
+}
+
 void OpenGLDriver::updateBufferObject(
         Handle<HwBufferObject> boh, BufferDescriptor&& bd, uint32_t byteOffset) {
     DEBUG_MARKER()
@@ -2564,6 +2597,19 @@ void OpenGLDriver::updateBufferObject(
         gl.bindVertexArray(nullptr);
     }
 
+    if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) {
+        auto streamDescriptors = mStreamUniformDescriptors.find(bo->gl.id);
+        if (streamDescriptors != mStreamUniformDescriptors.end()) {
+            for (auto const& [offset, stream, associationType] : streamDescriptors->second.mStreams) {
+                if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) {
+                    auto transform = getStreamTransformMatrix(stream);
+                    copyMat3f(bd.buffer, offset, transform);
+                }
+            }
+            mStreamUniformDescriptors.erase(streamDescriptors);
+        }
+    }
+
     if (UTILS_UNLIKELY(bo->bindingType == BufferObjectBinding::UNIFORM && gl.isES2())) {
         assert_invariant(bo->gl.buffer);
         memcpy(static_cast<uint8_t*>(bo->gl.buffer) + byteOffset, bd.buffer, bd.size);

filament/backend/src/opengl/OpenGLDriver.h

@@ -34,6 +34,7 @@
 #include <backend/Platform.h>
 #include <backend/Program.h>
 #include <backend/TargetBufferInfo.h>
+#include <backend/BufferObjectStreamDescriptor.h>
 
 #include "private/backend/Driver.h"
 #include "private/backend/HandleAllocator.h"
@@ -58,6 +59,7 @@
 #include <utility>
 #include <variant>
 #include <vector>
+#include <unordered_map>
 
 #include <stddef.h>
 #include <stdint.h>
@@ -364,6 +366,8 @@ private:
     // the must be accessed from the user thread only
     std::vector<GLStream*> mStreamsWithPendingAcquiredImage;
 
+    std::unordered_map<GLuint, BufferObjectStreamDescriptor> mStreamUniformDescriptors;
+
     void attachStream(GLTexture* t, GLStream* stream) noexcept;
     void detachStream(GLTexture* t) noexcept;
     void replaceStream(GLTexture* t, GLStream* stream) noexcept;

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -1148,6 +1148,10 @@ void VulkanDriver::updateIndexBuffer(Handle<HwIndexBuffer> ibh, BufferDescriptor
     scheduleDestroy(std::move(p));
 }
 
+void VulkanDriver::registerBufferObjectStreams(Handle<HwBufferObject> boh, BufferObjectStreamDescriptor&& streams) {
+    // Noop
+}
+
 void VulkanDriver::updateBufferObject(Handle<HwBufferObject> boh, BufferDescriptor&& bd,
         uint32_t byteOffset) {
     VulkanCommandBuffer& commands = mCommands.get();

filament/src/details/Engine.cpp

@@ -652,7 +652,8 @@ void FEngine::prepare() {
     DriverApi& driver = getDriverApi();
 
     for (auto& materialInstanceList: mMaterialInstances) {
-        materialInstanceList.second.forEach([&driver](FMaterialInstance const* item) {
+        materialInstanceList.second.forEach([&driver](FMaterialInstance* item) {
+            item->commitStreamUniformAssociations(driver);
             item->commit(driver);
         });
     }

filament/src/details/MaterialInstance.cpp

@@ -24,6 +24,7 @@
 #include "details/Material.h"
 #include "details/MaterialInstance.h"
 #include "details/Texture.h"
+#include "details/Stream.h"
 
 #include "private/filament/EngineEnums.h"
 
@@ -190,9 +191,27 @@ void FMaterialInstance::terminate(FEngine& engine) {
     driver.destroyBufferObject(mUbHandle);
 }
 
+void FMaterialInstance::commitStreamUniformAssociations(FEngine::DriverApi& driver) {
+    mHasStreamUniformAssociations = false;
+    if (!mTextureParameters.empty()) {
+        backend::BufferObjectStreamDescriptor descriptor;
+        for (auto const& [binding, p]: mTextureParameters) {
+            ssize_t offset = mMaterial->getUniformInterfaceBlock().getTransformFieldOffset(binding);
+            if (offset >= 0) {
+                mHasStreamUniformAssociations = true;
+                auto stream = p.texture->getStream()->getHandle();
+                descriptor.mStreams.push_back({uint32_t(offset), stream, BufferObjectStreamAssociationType::TRANSFORM_MATRIX});
+            }
+        }
+        if (descriptor.mStreams.size() > 0) {
+            driver.registerBufferObjectStreams(mUbHandle, std::move(descriptor));
+        }
+    }
+}
+
 void FMaterialInstance::commit(DriverApi& driver) const {
     // update uniforms if needed
-    if (mUniforms.isDirty()) {
+    if (mUniforms.isDirty() || mHasStreamUniformAssociations) {
         driver.updateBufferObject(mUbHandle, mUniforms.toBufferDescriptor(driver), 0);
     }
     if (!mTextureParameters.empty()) {

filament/src/details/MaterialInstance.h

@@ -65,6 +65,8 @@ public:
 
     void terminate(FEngine& engine);
 
+    void commitStreamUniformAssociations(FEngine::DriverApi& driver);
+    
     void commit(FEngine::DriverApi& driver) const;
 
     void use(FEngine::DriverApi& driver) const;
@@ -272,6 +274,7 @@ private:
     tsl::robin_map<backend::descriptor_binding_t, TextureParameter> mTextureParameters;
     mutable DescriptorSet mDescriptorSet;
     UniformBuffer mUniforms;
+    bool mHasStreamUniformAssociations = false;
 
     backend::PolygonOffset mPolygonOffset{};
     backend::StencilState mStencilState{};