diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4026192c5b..37b23ce26b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -11,6 +11,7 @@ A new header is inserted each time a *tag* is created. - Removed KTX1 compression support from mipgen. [⚠️ **API Change**]. - Added support for KTX2 to mipgen. - gltfio: Added resetBoneMatrices() method. +- gltfio: Introduced `TextureProvider` interface. [⚠️ **API Change**]. ## v1.21.2 diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java index 1fa0feaaed..4383c2161d 100644 --- a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java @@ -40,7 +40,7 @@ import com.google.android.filament.Renderer; * * When executing a test, clients should call tick() after each frame is rendered, which provides an * opportunity to push settings to Filament, increment the current test index (if enough time has - * elapsed), and request an asychronous screenshot. + * elapsed), and request an asynchronous screenshot. * * The time to sleep between tests is configurable and can be set to zero. Automation also waits a * specified minimum number of frames between tests. diff --git a/android/gltfio-android/CMakeLists.txt b/android/gltfio-android/CMakeLists.txt index b7a8c4471d..3a5b4f5f4f 100644 --- a/android/gltfio-android/CMakeLists.txt +++ b/android/gltfio-android/CMakeLists.txt @@ -11,6 +11,10 @@ add_library(dracodec STATIC IMPORTED) set_target_properties(dracodec PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libdracodec.a) +add_library(stb STATIC IMPORTED) +set_target_properties(stb PROPERTIES IMPORTED_LOCATION + ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libstb.a) + add_library(utils STATIC IMPORTED) set_target_properties(utils PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libutils.a) @@ -32,6 +36,7 @@ set(GLTFIO_SRCS ${GLTFIO_DIR}/include/gltfio/ResourceLoader.h ${GLTFIO_DIR}/include/gltfio/FilamentAsset.h ${GLTFIO_DIR}/include/gltfio/FilamentInstance.h + ${GLTFIO_DIR}/include/gltfio/TextureProvider.h ${GLTFIO_DIR}/include/gltfio/math.h ${GLTFIO_DIR}/src/Animator.cpp @@ -47,13 +52,13 @@ set(GLTFIO_SRCS ${GLTFIO_DIR}/src/GltfEnums.h ${GLTFIO_DIR}/src/MaterialProvider.cpp ${GLTFIO_DIR}/src/ResourceLoader.cpp + ${GLTFIO_DIR}/src/StbProvider.cpp ${GLTFIO_DIR}/src/TangentsJob.h ${GLTFIO_DIR}/src/TangentsJob.cpp ${GLTFIO_DIR}/src/UbershaderLoader.cpp ${GLTFIO_DIR}/src/Wireframe.cpp ${GLTFIO_DIR}/src/Wireframe.h ${GLTFIO_DIR}/src/upcast.h - ${GLTFIO_DIR}/src/Image.cpp src/main/cpp/Animator.cpp src/main/cpp/AssetLoader.cpp @@ -90,9 +95,9 @@ set_target_properties(gltfio-jni PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_ if(GLTFIO_LITE) target_compile_definitions(gltfio-jni PUBLIC GLTFIO_LITE=1) - target_link_libraries(gltfio-jni filament-jni utils log gltfio_resources_lite) + target_link_libraries(gltfio-jni filament-jni utils log stb gltfio_resources_lite) else() - target_link_libraries(gltfio-jni filament-jni utils log gltfio_resources) + target_link_libraries(gltfio-jni filament-jni utils log stb gltfio_resources) # Enable Draco in the non-lite variant of gltfio. target_link_libraries(gltfio-jni dracodec) diff --git a/android/gltfio-android/src/main/cpp/ResourceLoader.cpp b/android/gltfio-android/src/main/cpp/ResourceLoader.cpp index c16ba604ed..d1a48a9750 100644 --- a/android/gltfio-android/src/main/cpp/ResourceLoader.cpp +++ b/android/gltfio-android/src/main/cpp/ResourceLoader.cpp @@ -19,6 +19,7 @@ #include #include +#include #include @@ -114,3 +115,27 @@ Java_com_google_android_filament_gltfio_ResourceLoader_nAsyncCancelLoad(JNIEnv*, ResourceLoader* loader = (ResourceLoader*) nativeLoader; loader->asyncCancelLoad(); } + +extern "C" JNIEXPORT jlong JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nCreateTextureProvider(JNIEnv*, jclass, + jlong nativeEngine) { + Engine* engine = (Engine*) nativeEngine; + return (jlong) createStbProvider(engine); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nDestroyTextureProvider(JNIEnv*, jclass, + jlong nativeProvider) { + TextureProvider* provider = (TextureProvider*) nativeProvider; + delete provider; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nAddTextureProvider(JNIEnv* env, jclass, + jlong nativeLoader, jstring url, jlong nativeProvider) { + ResourceLoader* loader = (ResourceLoader*) nativeLoader; + TextureProvider* provider = (TextureProvider*) nativeProvider; + const char* cstring = env->GetStringUTFChars(url, nullptr); + loader->addTextureProvider(cstring, provider); + env->ReleaseStringUTFChars(url, cstring); +} diff --git a/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java b/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java index 976a752caf..217909c2ff 100644 --- a/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java +++ b/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java @@ -35,6 +35,7 @@ import java.nio.Buffer; */ public class ResourceLoader { private final long mNativeObject; + private final long mNativeProvider; /** * Constructs a resource loader tied to the given Filament engine. @@ -46,6 +47,9 @@ public class ResourceLoader { public ResourceLoader(@NonNull Engine engine) { long nativeEngine = engine.getNativeObject(); mNativeObject = nCreateResourceLoader(nativeEngine, false, false, false); + mNativeProvider = nCreateTextureProvider(nativeEngine); + nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider); + nAddTextureProvider(mNativeObject, "image/png", mNativeProvider); } /** @@ -63,6 +67,9 @@ public class ResourceLoader { long nativeEngine = engine.getNativeObject(); mNativeObject = nCreateResourceLoader(nativeEngine, normalizeSkinningWeights, recomputeBoundingBoxes, ignoreBindTransform); + mNativeProvider = nCreateTextureProvider(nativeEngine); + nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider); + nAddTextureProvider(mNativeObject, "image/png", mNativeProvider); } /** @@ -70,6 +77,7 @@ public class ResourceLoader { */ public void destroy() { nDestroyResourceLoader(mNativeObject); + nDestroyTextureProvider(mNativeObject); } /** @@ -178,4 +186,8 @@ public class ResourceLoader { private static native float nAsyncGetLoadProgress(long nativeLoader); private static native void nAsyncUpdateLoad(long nativeLoader); private static native void nAsyncCancelLoad(long nativeLoader); + + private static native long nCreateTextureProvider(long nativeEngine); + private static native void nAddTextureProvider(long nativeLoader, String url, long nativeProvider); + private static native void nDestroyTextureProvider(long nativeProvider); } diff --git a/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm b/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm index 8c2e98b547..75f5e2de98 100644 --- a/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm +++ b/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -78,6 +79,7 @@ const float kSensitivity = 100.0f; ResourceLoader* _resourceLoader; Manipulator* _manipulator; + TextureProvider* _decoder; FilamentAsset* _asset; @@ -129,6 +131,9 @@ const float kSensitivity = 100.0f; _assetLoader = AssetLoader::create({_engine, _materialProvider, ncm, &em}); _resourceLoader = new ResourceLoader( {.engine = _engine, .normalizeSkinningWeights = true, .recomputeBoundingBoxes = false}); + _decoder = createStbProvider(_engine); + _resourceLoader->addTextureProvider("image/png", _decoder); + _resourceLoader->addTextureProvider("image/jpeg", _decoder); _manipulator = Manipulator::Builder().orbitHomePosition(0.0f, 0.0f, 4.0f).build(Mode::ORBIT); @@ -247,6 +252,7 @@ const float kSensitivity = 100.0f; [self destroyModel]; delete _manipulator; + delete _decoder; _materialProvider->destroyMaterials(); delete _materialProvider; diff --git a/libs/filamentapp/CMakeLists.txt b/libs/filamentapp/CMakeLists.txt index 7414278ce9..5855d0d032 100644 --- a/libs/filamentapp/CMakeLists.txt +++ b/libs/filamentapp/CMakeLists.txt @@ -33,7 +33,6 @@ set(SRCS src/FilamentApp.cpp src/IBL.cpp src/IcoSphere.cpp - src/Image.cpp src/MeshAssimp.cpp src/Sphere.cpp ) diff --git a/libs/filamentapp/src/Image.cpp b/libs/filamentapp/src/Image.cpp deleted file mode 100644 index 38a8c06c2b..0000000000 --- a/libs/filamentapp/src/Image.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_TGA -#define STBI_NO_GIF -#define STBI_NO_PIC -#define STBI_NO_PNM - -#define STB_IMAGE_IMPLEMENTATION -#include diff --git a/libs/gltfio/CMakeLists.txt b/libs/gltfio/CMakeLists.txt index 1ee2113135..9fe414a458 100644 --- a/libs/gltfio/CMakeLists.txt +++ b/libs/gltfio/CMakeLists.txt @@ -14,6 +14,7 @@ set(PUBLIC_HDRS include/gltfio/ResourceLoader.h include/gltfio/FilamentAsset.h include/gltfio/FilamentInstance.h + include/gltfio/TextureProvider.h include/gltfio/math.h ) @@ -31,13 +32,13 @@ set(SRCS src/GltfEnums.h src/MaterialProvider.cpp src/ResourceLoader.cpp + src/StbProvider.cpp src/TangentsJob.h src/TangentsJob.cpp src/UbershaderLoader.cpp src/Wireframe.cpp src/Wireframe.h src/upcast.h - src/Image.cpp ) # ================================================================================================== @@ -196,10 +197,7 @@ if (NOT WEBGL AND NOT ANDROID AND NOT IOS) # Installation # ================================================================================================== install(TARGETS ${TARGET} gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) - # The Image.h header is only needed within Filament and does not need to be distributed. - install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio - DESTINATION include - PATTERN "Image.h" EXCLUDE) + install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio DESTINATION include) else() install(TARGETS gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) diff --git a/libs/gltfio/include/gltfio/AssetLoader.h b/libs/gltfio/include/gltfio/AssetLoader.h index 81feaeb23d..90491c9852 100644 --- a/libs/gltfio/include/gltfio/AssetLoader.h +++ b/libs/gltfio/include/gltfio/AssetLoader.h @@ -84,6 +84,7 @@ struct AssetConfiguration { * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * auto engine = Engine::create(); * auto materials = createMaterialGenerator(engine); + * auto decoder = createStbProvider(engine); * auto loader = AssetLoader::create({engine, materials}); * * // Parse the glTF content and create Filament entities. @@ -92,7 +93,10 @@ struct AssetConfiguration { * content.clear(); * * // Load buffers and textures from disk. - * ResourceLoader({engine, ".", true}).loadResources(asset); + * ResourceLoader resourceLoader({engine, ".", true}); + * resourceLoader.addTextureProvider("image/png", decoder) + * resourceLoader.addTextureProvider("image/jpeg", decoder) + * resourceLoader.loadResources(asset); * * // Free the glTF hierarchy as it is no longer needed. * asset->releaseSourceData(); @@ -114,6 +118,7 @@ struct AssetConfiguration { * loader->destroyAsset(asset); * materials->destroyMaterials(); * delete materials; + * delete decoder; * AssetLoader::destroy(&loader); * Engine::destroy(&engine); * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/libs/gltfio/include/gltfio/Image.h b/libs/gltfio/include/gltfio/Image.h deleted file mode 100644 index c7047311d2..0000000000 --- a/libs/gltfio/include/gltfio/Image.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// gltfio supports PNG and JPEG, disable all other formats. -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_TGA -#define STBI_NO_GIF -#define STBI_NO_HDR -#define STBI_NO_PIC -#define STBI_NO_PNM - -// For emscripten and Android builds, we never load from the file -// system, so we-opt out of the stdio functionality in stb. -#if defined(__EMSCRIPTEN__) || defined(__ANDROID__) -#define STBI_NO_STDIO -#endif - -#include diff --git a/libs/gltfio/include/gltfio/ResourceLoader.h b/libs/gltfio/include/gltfio/ResourceLoader.h index a92a4f8517..9de8bc2f25 100644 --- a/libs/gltfio/include/gltfio/ResourceLoader.h +++ b/libs/gltfio/include/gltfio/ResourceLoader.h @@ -31,6 +31,7 @@ namespace gltfio { struct FFilamentAsset; class AssetPool; +class TextureProvider; /** * \struct ResourceConfiguration ResourceLoader.h gltfio/ResourceLoader.h @@ -94,6 +95,13 @@ public: */ void addResourceData(const char* uri, BufferDescriptor&& buffer); + /** + * Register a plugin that can consume PNG / JPEG content and produce filament::Texture objects. + * + * Destruction of the given provider is the client's responsibility. + */ + void addTextureProvider(const char* mimeType, TextureProvider* provider); + /** * Checks if the given resource has already been added to the URI cache. */ diff --git a/libs/gltfio/include/gltfio/TextureProvider.h b/libs/gltfio/include/gltfio/TextureProvider.h new file mode 100644 index 0000000000..3bdbba0d80 --- /dev/null +++ b/libs/gltfio/include/gltfio/TextureProvider.h @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GLTFIO_TEXTUREPROVIDER_H +#define GLTFIO_TEXTUREPROVIDER_H + +#include +#include + +#include + +namespace filament { + class Engine; + class Texture; +} + +namespace gltfio { + +/** + * TextureProvider is an interface that allows clients to implement their own texture decoding + * facility for JPEG, PNG, or KTX2 content. It constructs Filament Texture objects synchronously, + * but populates their miplevels asynchronously. + * + * gltfio calls all public methods from the foreground thread, i.e. the thread that the Filament + * engine was created with. However the implementation may create 0 or more background threads to + * perform decoding work. + * + * The following pseudocode illustrates how this interface could be used, but in practice the only + * client is the gltfio ResourceLoader. + * + * filament::Engine* engine = ...; + * TextureProvider* provider = createStbProvider(engine); + * + * for (auto filename : textureFiles) { + * std::vector buf = readEntireFile(filename); + * Texture* texture = provider->pushTexture(buf.data(), buf.size(), "image/png", 0); + * if (texture == nullptr) { puts(provider->getPushMessage()); exit(1); } + * } + * + * // At this point, the returned textures can be bound to material instances, but none of their + * // miplevel images have been populated yet. + * + * while (provider->getPoppedCount() < provider->getPushedCount()) { + * sleep(200); + * + * // The following call gives the provider an opportunity to reap the results of any + * // background decoder work that has been completed (e.g. by calling Texture::setImage). + * provider->updateQueue(); + * + * // Check for textures that now have all their miplevels initialized. + * while (Texture* texture = provider->popTexture()) { + * printf("%p has all its miplevels ready.\n", texture); + * } + * } + * + * delete provider; + */ +class UTILS_PUBLIC TextureProvider { +public: + using Texture = filament::Texture; + using FlagBits = uint64_t; + + enum class Flags { + sRGB = 1 << 0, + }; + + /** + * Creates a Filament texture and pushes it to the asynchronous decoding queue. + * + * If construction fails, nothing is pushed to the queue and null is returned. The failure + * reason can be obtained with getPushMessage(). The given buffer pointer is not held, so the + * caller can free it immediately. It is also the caller's responsibility to free the returned + * Texture object, but it is only safe to do so after it has been popped from the queue. + */ + virtual Texture* pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, FlagBits flags) = 0; + + /** + * Checks if any texture is ready to be removed from the asynchronous decoding queue, and if so + * pops it off. + * + * Unless an error or cancellation occurred during the decoding process, the returned texture + * should have all its miplevels populated. If the texture is not complete, the reason can be + * obtained with getPopMessage(). + * + * Due to concurrency, textures are not necessarily popped off in the same order they were + * pushed. Returns null if there are no textures that are ready to be popped. + */ + virtual Texture* popTexture() = 0; + + /** + * Polls textures in the queue and uploads mipmap images if any have emerged from the decoder. + * + * This gives the provider an opportunity to call Texture::setImage() on the foreground thread. + * If needed, it can also call Texture::generateMipmaps() here. + * + * Items in the decoding queue can become "poppable" only during this call. + */ + virtual void updateQueue() = 0; + + /** + * Returns a failure message for the most recent call to pushTexture(), or null for success. + * + * Note that this method does not pertain to the decoding process. If decoding fails, clients to + * can pop the incomplete texture off the queue and obtain a failure message using the + * getPopFailure() method. + * + * The returned string is owned by the provider and becomes invalid after the next call to + * pushTexture(). + */ + virtual const char* getPushMessage() const = 0; + + /** + * Returns a failure message for the most recent call to popTexture(), or null for success. + * + * If the most recent call to popTexture() returned null, then no error occurred and this + * returns null. If the most recent call to popTexture() returned a "complete" texture (i.e. + * all miplevels present), then this returns null. This returns non-null only if an error or + * cancellation occurred while decoding the popped texture. + * + * The returned string is owned by the provider and becomes invalid after the next call to + * popTexture(). + */ + virtual const char* getPopMessage() const = 0; + + /** + * Waits for all outstanding decoding jobs to complete. + * + * Clients should call updateQueue() afterwards if they wish to update the push / pop queue. + */ + virtual void waitForCompletion() = 0; + + /** + * Cancels all not-yet-started decoding jobs and waits for all other jobs to complete. + * + * Jobs that have already started cannot be canceled. Textures whose decoding process has + * been cancelled will be made poppable on the subsequent call to updateQueue(). + */ + virtual void cancelDecoding() = 0; + + /** Total number of successful push calls since the provider was created. */ + virtual size_t getPushedCount() const = 0; + + /** Total number of successful pop calls since the provider was created. */ + virtual size_t getPoppedCount() const = 0; + + /** Total number of textures that have become ready-to-pop since the provider was created. */ + virtual size_t getDecodedCount() const = 0; + + virtual ~TextureProvider() = default; +}; + +/** + * Creates a simple decoder based on stb_image that can handle "image/png" and "image/jpeg". + * This works only if your build configuration includes STB. + */ +TextureProvider* createStbProvider(filament::Engine* engine); + +} // namespace gltfio + +#endif // GLTFIO_TEXTUREPROVIDER_H diff --git a/libs/gltfio/src/DependencyGraph.cpp b/libs/gltfio/src/DependencyGraph.cpp index 24812b03c3..9a92dbf7c6 100644 --- a/libs/gltfio/src/DependencyGraph.cpp +++ b/libs/gltfio/src/DependencyGraph.cpp @@ -75,7 +75,7 @@ void DependencyGraph::refinalize() { } void DependencyGraph::addEdge(Texture* texture, MaterialInstance* material, const char* parameter) { - assert(mFinalized); + assert(texture && !mFinalized); mTextureToMaterial[texture].insert(material); mMaterialToTexture.at(material).params.at(parameter) = getStatus(texture); } @@ -86,6 +86,7 @@ void DependencyGraph::checkReadiness(Material* material) { // Check this material's texture parameters, there are 5 in the worst case. bool materialIsReady = true; for (auto pair : status.params) { + assert(pair.second && "Parameter-to-Texture edge is missing."); if (!pair.second->ready) { materialIsReady = false; break; @@ -125,6 +126,7 @@ void DependencyGraph::markAsReady(MaterialInstance* material) { } DependencyGraph::TextureNode* DependencyGraph::getStatus(Texture* texture) { + assert(texture); auto iter = mTextureNodes.find(texture); if (iter == mTextureNodes.end()) { TextureNode* status = (mTextureNodes[texture] = std::make_unique()).get(); diff --git a/libs/gltfio/src/DependencyGraph.h b/libs/gltfio/src/DependencyGraph.h index 854571c2f4..b391cbc109 100644 --- a/libs/gltfio/src/DependencyGraph.h +++ b/libs/gltfio/src/DependencyGraph.h @@ -63,6 +63,7 @@ public: using Entity = utils::Entity; // Pops up to "count" ready-to-render entities off the queue. + // // If "result" is non-null, returns the number of written items. // If "result" is null, returns the number of available entities. size_t popRenderables(Entity* result, size_t count) noexcept; @@ -70,19 +71,27 @@ public: // These are called during the initial asset loader phase. void addEdge(Entity entity, Material* material); void addEdge(Material* material, const char* parameter); + void addEdge(filament::Texture* texture, Material* material, const char* parameter); - // This is called at the end of the initial asset loading phase. - // Makes a guarantee that no new material nodes or parameter nodes will be added to the graph. + // Marks the end of synchronous asset loading. + // + // At this point, the graph enters a finalized state and all non-textured entities are + // immediately marked as "ready". However textures are not yet fully decoded. + // + // After finalization, the only nodes that can be added to the graph are entities. void finalize(); - // This can be called after finalization to allow for dynamic addition of entities. - // It is slower than finalize() because it checks the readiness of existing materials. - void refinalize(); - - // These are called after textures have created and decoded. - void addEdge(filament::Texture* texture, Material* material, const char* parameter); + // Marks the given texture as being fully decoded, with all miplevels initialized. + // + // This can only be called on a finalized graph. void markAsReady(filament::Texture* texture); + // Re-checks the readiness of all entities after finalization. + // + // This exists only to support dynamic instancing. It is slower than finalize() because it + // checks the readiness of existing materials. + void refinalize(); + private: struct TextureNode { filament::Texture* texture; diff --git a/libs/gltfio/src/Image.cpp b/libs/gltfio/src/Image.cpp deleted file mode 100644 index 815649b5ec..0000000000 --- a/libs/gltfio/src/Image.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define STB_IMAGE_IMPLEMENTATION - -#include diff --git a/libs/gltfio/src/ResourceLoader.cpp b/libs/gltfio/src/ResourceLoader.cpp index 5fbe74e6cf..47737287ae 100644 --- a/libs/gltfio/src/ResourceLoader.cpp +++ b/libs/gltfio/src/ResourceLoader.cpp @@ -15,7 +15,7 @@ */ #include -#include +#include #include "GltfEnums.h" #include "FFilamentAsset.h" @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -45,12 +46,12 @@ #include #include +#include #if defined(__EMSCRIPTEN__) || defined(__ANDROID__) || defined(IOS) #define USE_FILESYSTEM 0 #else #define USE_FILESYSTEM 1 -#include #endif using namespace filament; @@ -62,25 +63,13 @@ using filament::geometry::ComponentType; static const auto FREE_CALLBACK = [](void* mem, size_t, void*) { free(mem); }; -namespace { - struct TextureCacheEntry { - Texture* texture; - std::atomic texels; - uint32_t bufferSize; - int width; - int height; - int numComponents; - bool srgb; - bool completed; - }; - - using BufferTextureCache = tsl::robin_map>; - using UriTextureCache = tsl::robin_map>; - using UriDataCache = tsl::robin_map; -} - namespace gltfio { +using BufferTextureCache = tsl::robin_map; +using FilepathTextureCache = tsl::robin_map; +using UriDataCache = tsl::robin_map; +using TextureProviderList = tsl::robin_map; + struct ResourceLoader::Impl { Impl(const ResourceConfiguration& config) { mGltfPath = std::string(config.gltfPath ? config.gltfPath : ""); @@ -96,33 +85,27 @@ struct ResourceLoader::Impl { bool mIgnoreBindTransform; std::string mGltfPath; - // This is used to calculate skinIndex when updateBoundingBoxes, so that the correspondency between + // This is used to calculate skinIndex when updateBoundingBoxes, so that the mapping between // cgltf_node* and FFilamentInstance::Skin can be retrieved. This pointer doesn't need to be freed. - cgltf_skin* cgltfSkinBaseAddress; + cgltf_skin* mSkinBaseAddress; // User-provided resource data with URI string keys, populated with addResourceData(). // This is used on platforms without traditional file systems, such as Android, iOS, and WebGL. UriDataCache mUriDataCache; - // The two texture caches are populated while textures are being decoded, and they are no longer - // used after all textures have been finalized. Since multiple glTF textures might be loaded - // from a single URI or buffer pointer, these caches prevent needless re-decoding. There are - // two caches: one for URI-based textures and one for buffer-based textures. + // User-provided mapping from mime types to texture providers. + TextureProviderList mTextureProviders; + + // Avoid duplicated Texture objects via caches with two key types: buffer pointers and strings. BufferTextureCache mBufferTextureCache; - UriTextureCache mUriTextureCache; - int mNumDecoderTasks; - int mNumDecoderTasksFinished; - JobSystem::Job* mDecoderRootJob = nullptr; - FFilamentAsset* mCurrentAsset = nullptr; + FilepathTextureCache mFilepathTextureCache; + + FFilamentAsset* mAsyncAsset = nullptr; void computeTangents(FFilamentAsset* asset); - bool createTextures(bool async); + bool createTextures(FFilamentAsset* asset, bool async); void cancelTextureDecoding(); - void addTextureCacheEntry(const TextureSlot& tb); - void bindTextureToMaterial(const TextureSlot& tb); - void decodeSingleTexture(); - void uploadPendingTextures(); - void releasePendingTextures(); + Texture* getOrCreateTexture(FFilamentAsset* asset, const TextureSlot& tb); ~Impl(); }; @@ -393,7 +376,8 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { if (strncmp(uri, "data:", 5) == 0) { const char* comma = strchr(uri, ','); if (comma && comma - uri >= 7 && strncmp(comma - 7, ";base64", 7) == 0) { - cgltf_result res = cgltf_load_buffer_base64(&options, gltf->buffers[i].size, comma + 1, &gltf->buffers[i].data); + cgltf_result res = cgltf_load_buffer_base64(&options, gltf->buffers[i].size, + comma + 1, &gltf->buffers[i].data); if (res != cgltf_result_success) { slog.e << "Unable to load " << uri << io::endl; return false; @@ -469,7 +453,7 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { if (!pImpl->mIgnoreBindTransform) { pImpl->mIgnoreBindTransform = asset->isInstanced(); } - pImpl->cgltfSkinBaseAddress = &gltf->skins[0]; + pImpl->mSkinBaseAddress = &gltf->skins[0]; updateBoundingBoxes(asset); } @@ -533,12 +517,12 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { // we need to generate the contents of a GPU buffer by processing one or more CPU buffer(s). pImpl->computeTangents(asset); - // Non-textured renderables are now considered ready, so notify the dependency graph. - asset->mDependencyGraph.finalize(); - pImpl->mCurrentAsset = asset; - // Finally, create Filament Textures and begin loading image files. - asset->mResourcesLoaded = pImpl->createTextures(async); + asset->mResourcesLoaded = pImpl->createTextures(asset, async); + + // Non-textured renderables are now considered ready, and we can guarantee that no new + // materials or textures will be added. notify the dependency graph. + asset->mDependencyGraph.finalize(); asset->createAnimators(); @@ -546,334 +530,178 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { } bool ResourceLoader::asyncBeginLoad(FilamentAsset* asset) { + pImpl->mAsyncAsset = upcast(asset); return loadResources(upcast(asset), true); } void ResourceLoader::asyncCancelLoad() { pImpl->cancelTextureDecoding(); + pImpl->mAsyncAsset = nullptr; pImpl->mEngine->flushAndWait(); } +void ResourceLoader::addTextureProvider(const char* mimeType, TextureProvider* provider) { + pImpl->mTextureProviders[mimeType] = provider; +} + float ResourceLoader::asyncGetLoadProgress() const { - const float finished = pImpl->mNumDecoderTasksFinished; - const float total = pImpl->mNumDecoderTasks; - return total == 0 ? 0 : finished / total; + if (pImpl->mTextureProviders.empty() || !pImpl->mAsyncAsset) { + return 0; + } + size_t pushedCount = 0; + size_t decodedCount = 0; + for (const auto& iter : pImpl->mTextureProviders) { + pushedCount += iter.second->getPushedCount(); + decodedCount += iter.second->getDecodedCount(); + } + return pushedCount == 0 ? 0 : (float(decodedCount) / pushedCount); } void ResourceLoader::asyncUpdateLoad() { - if (!UTILS_HAS_THREADING) { - pImpl->decodeSingleTexture(); - } - pImpl->uploadPendingTextures(); -} - -void ResourceLoader::Impl::decodeSingleTexture() { - assert(!UTILS_HAS_THREADING); - int w, h, c; - - // Check if any buffer-based textures haven't been decoded yet. - for (auto& pair : mBufferTextureCache) { - const uint8_t* sourceData = (const uint8_t*) pair.first; - TextureCacheEntry* entry = pair.second.get(); - if (entry->texels) { - continue; - } - entry->texels = stbi_load_from_memory(sourceData, entry->bufferSize, &w, &h, &c, 4); + if (!pImpl->mAsyncAsset) { return; } - - // Check if any URI-based textures haven't been decoded yet. - for (auto& pair : mUriTextureCache) { - auto uri = pair.first; - TextureCacheEntry* entry = pair.second.get(); - if (entry->texels) { - continue; + for (const auto& iter : pImpl->mTextureProviders) { + iter.second->updateQueue(); + while (Texture* texture = iter.second->popTexture()) { + pImpl->mAsyncAsset->mDependencyGraph.markAsReady(texture); } - - // First, check the user-supplied resource cache for this URI. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - entry->texels = stbi_load_from_memory(sourceData, iter->second.size, &w, &h, &c, 4); - return; - } - - // Otherwise load it from the file system if this platform supports it. - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - entry->completed = true; - mNumDecoderTasksFinished++; - return; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - entry->texels = stbi_load(fullpath.c_str(), &w, &h, &c, 4); - return; - #endif } } -void ResourceLoader::Impl::uploadPendingTextures() { - auto upload = [this](TextureCacheEntry* entry, Engine& engine) { - Texture* texture = entry->texture; - uint8_t* texels = entry->texels; - if (texture && texels && !entry->completed) { - Texture::PixelBufferDescriptor pbd(texels, - texture->getWidth() * texture->getHeight() * 4, - Texture::Format::RGBA, Texture::Type::UBYTE, FREE_CALLBACK); - texture->setImage(engine, 0, std::move(pbd)); - texture->generateMipmaps(engine); - entry->completed = true; - mNumDecoderTasksFinished++; - mCurrentAsset->mDependencyGraph.markAsReady(texture); - } - }; - for (auto& pair : mBufferTextureCache) upload(pair.second.get(), *mEngine); - for (auto& pair : mUriTextureCache) upload(pair.second.get(), *mEngine); -} - -void ResourceLoader::Impl::releasePendingTextures() { - auto release = [this](TextureCacheEntry* entry, Engine& engine) { - Texture* texture = entry->texture; - uint8_t* texels = entry->texels; - if (texture && texels && !entry->completed) { - // Normally the ownership of these texels is transferred to PixelBufferDescriptor, but - // if uploads have been cancelled then we need to free them explicitly. - free(texels); - } - }; - for (auto& pair : mBufferTextureCache) release(pair.second.get(), *mEngine); - for (auto& pair : mUriTextureCache) release(pair.second.get(), *mEngine); -} - -void ResourceLoader::Impl::addTextureCacheEntry(const TextureSlot& tb) { - TextureCacheEntry* entry = nullptr; - +Texture* ResourceLoader::Impl::getOrCreateTexture(FFilamentAsset* asset, const TextureSlot& tb) { const cgltf_texture* srcTexture = tb.texture; - const cgltf_buffer_view* bv = srcTexture->image->buffer_view; - const char* uri = srcTexture->image->uri; - const uint32_t totalSize = uint32_t(bv ? bv->size : 0); - void** data = bv ? &bv->buffer->data : nullptr; - const size_t offset = bv ? bv->offset : 0; + const cgltf_image* image = srcTexture->image; + const cgltf_buffer_view* bv = image->buffer_view; + const char* uri = image->uri; - // Check if the texture binding uses BufferView data (i.e. it does not have a URI). - if (data) { - const uint8_t* sourceData = offset + (const uint8_t*) *data; - entry = mBufferTextureCache[sourceData] ? mBufferTextureCache[sourceData].get() : nullptr; - if (entry) { - return; - } - entry = (mBufferTextureCache[sourceData] = std::make_unique()).get(); - entry->srgb = tb.srgb; - if (!stbi_info_from_memory(sourceData, totalSize, &entry->width, &entry->height, - &entry->numComponents)) { - slog.e << "Unable to decode BufferView texture: " << stbi_failure_reason() << io::endl; - mBufferTextureCache.erase(sourceData); - return; - } - entry->bufferSize = totalSize; - return; + TextureProvider::FlagBits flags = {}; + if (tb.srgb) { + flags |= int(TextureProvider::Flags::sRGB); } - // Check if we already created a Texture object for this URI. - entry = mUriTextureCache[uri] ? mUriTextureCache[uri].get() : nullptr; - if (entry) { - return; - } - - entry = (mUriTextureCache[uri] = std::make_unique()).get(); - entry->srgb = tb.srgb; - - // Check if this is a data URI. We don't care about the MIME type since stb can infer it. - std::string mimeType; + std::string mime = image->mime_type ? image->mime_type : ""; size_t dataUriSize; - const uint8_t* dataUriContent = parseDataUri(uri, &mimeType, &dataUriSize); - if (dataUriContent) { - BufferDescriptor buffer(dataUriContent, dataUriSize, FREE_CALLBACK); - mUriDataCache.emplace(uri, std::move(buffer)); + const uint8_t* dataUriContent = uri ? parseDataUri(uri, &mime, &dataUriSize) : nullptr; + + if (mime.empty()) { + assert_invariant(uri && "Non-URI images must supply a mime type."); + const std::string extension = Path(uri).getExtension(); + mime = extension == "jpg" ? "image/jpeg" : "image/" + extension; } - // Check the user-supplied resource cache for this URI, otherwise peek at the file. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - if (!stbi_info_from_memory(sourceData, iter->second.size, &entry->width, - &entry->height, &entry->numComponents)) { - slog.e << "Unable to decode " << uri << " : " << stbi_failure_reason() << io::endl; - mUriTextureCache.erase(uri); - } - return; + TextureProvider* provider = mTextureProviders[mime]; + if (!provider) { + slog.e << "Missing texture provider for " << mime << io::endl; + return nullptr; } - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - if (!stbi_info(fullpath.c_str(), &entry->width, &entry->height, &entry->numComponents)) { - slog.e << "Unable to decode " << fullpath.c_str() << " : " << stbi_failure_reason() - << io::endl; - mUriTextureCache.erase(uri); - } - #endif -} -void ResourceLoader::Impl::bindTextureToMaterial(const TextureSlot& tb) { - FFilamentAsset* asset = mCurrentAsset; + Texture* texture = nullptr; - const cgltf_texture* srcTexture = tb.texture; - const cgltf_buffer_view* bv = srcTexture->image->buffer_view; - const char* uri = srcTexture->image->uri; - void** data = bv ? &bv->buffer->data : nullptr; - const size_t offset = bv ? bv->offset : 0; - - // First check if this is a buffer-based texture. - if (data) { - const uint8_t* sourceData = offset + (const uint8_t*) *data; + // Check if the texture slot uses BufferView data. + if (void** bufferViewData = bv ? &bv->buffer->data : nullptr; bufferViewData) { + assert_invariant(!dataUriContent); + const size_t offset = bv ? bv->offset : 0; + const uint8_t* sourceData = offset + (const uint8_t*) *bufferViewData; if (auto iter = mBufferTextureCache.find(sourceData); iter != mBufferTextureCache.end()) { - auto& entry = iter->second; - if (entry.get() && entry->texture) { - asset->bindTexture(tb, entry->texture); - } + return iter->second; + } + const uint32_t totalSize = uint32_t(bv ? bv->size : 0); + if ((texture = provider->pushTexture(sourceData, totalSize, mime.c_str(), flags))) { + mBufferTextureCache[sourceData] = texture; } - return; } - // Next check if this is a URI-based texture. - if (auto iter = mUriTextureCache.find(uri); iter != mUriTextureCache.end()) { - auto& entry = iter->second; - if (entry.get() && entry->texture) { - asset->bindTexture(tb, entry->texture); + // Check if the texture slot is a data URI. + else if (dataUriContent) { + if (auto iter = mBufferTextureCache.find(uri); iter != mBufferTextureCache.end()) { + free((void*)dataUriContent); + return iter->second; + } + if ((texture = provider->pushTexture(dataUriContent, dataUriSize, mime.c_str(), flags))) { + mBufferTextureCache[uri] = texture; + } + free((void*)dataUriContent); + } + + // Check the user-supplied resource cache for this URI. + else if (auto iter = mUriDataCache.find(uri); iter != mUriDataCache.end()) { + const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; + if (auto iter = mBufferTextureCache.find(sourceData); iter != mBufferTextureCache.end()) { + return iter->second; + } + if ((texture = provider->pushTexture(sourceData, iter->second.size, mime.c_str(), flags))) { + mBufferTextureCache[sourceData] = texture; } } + + // Finally, try the file system. + else if constexpr (USE_FILESYSTEM) { + if (auto iter = mFilepathTextureCache.find(uri); iter != mFilepathTextureCache.end()) { + return iter->second; + } + Path fullpath = Path(mGltfPath).getParent() + uri; + if (!fullpath.exists()) { + slog.e << "Unable to open " << fullpath << io::endl; + return nullptr; + } + using namespace std; + ifstream filest(fullpath); + vector buffer; + filest.seekg(0, ios::end); + buffer.reserve((size_t) filest.tellg()); + filest.seekg(0, ios::beg); + buffer.assign((istreambuf_iterator(filest)), istreambuf_iterator()); + if ((texture = provider->pushTexture(buffer.data(), buffer.size(), mime.c_str(), flags))) { + mFilepathTextureCache[uri] = texture; + } + + // If the platform does not have a filesystem, emit an error and move on. + } else { + slog.e << "Unable to load " << uri << io::endl; + return nullptr; + } + + if (!texture) { + slog.e << "Unable to decode texture data: " << provider->getPushMessage() << io::endl; + } + + asset->takeOwnership(texture); + return texture; } void ResourceLoader::Impl::cancelTextureDecoding() { - JobSystem* js = &mEngine->getJobSystem(); - if (mDecoderRootJob) { - js->waitAndRelease(mDecoderRootJob); - mDecoderRootJob = nullptr; + for (const auto& iter : mTextureProviders) { + iter.second->cancelDecoding(); } - releasePendingTextures(); - mBufferTextureCache.clear(); - mUriTextureCache.clear(); - mCurrentAsset = nullptr; - mNumDecoderTasksFinished = 0; - mNumDecoderTasks = 0; + mAsyncAsset = nullptr; } -bool ResourceLoader::Impl::createTextures(bool async) { +bool ResourceLoader::Impl::createTextures(FFilamentAsset* asset, bool async) { // If any decoding jobs are still underway, wait for them to finish. - JobSystem* js = &mEngine->getJobSystem(); - if (mDecoderRootJob) { - js->waitAndRelease(mDecoderRootJob); - mDecoderRootJob = nullptr; + for (const auto& iter : mTextureProviders) { + iter.second->waitForCompletion(); + iter.second->updateQueue(); } - mBufferTextureCache.clear(); - mUriTextureCache.clear(); - - // First, determine texture dimensions and create texture cache entries. - FFilamentAsset* asset = mCurrentAsset; + // Create new texture objects if they are not cached. for (auto slot : asset->mTextureSlots) { - addTextureCacheEntry(slot); - } - - // Tally up the total number of textures that need to be decoded. Zero textures is a special - // case that needs to report 100% progress right away, so we set NumDecoderTasks and Finished - // both to 1. If they were both 0, this would indicate that loading has not started. - mNumDecoderTasks = mBufferTextureCache.size() + mUriTextureCache.size(); - if (mNumDecoderTasks == 0) { - mNumDecoderTasks = 1; - mNumDecoderTasksFinished = 1; - } else { - mNumDecoderTasksFinished = 0; - } - - // Next create blank Filament textures. - auto createTexture = [=](TextureCacheEntry* entry) { - entry->texture = Texture::Builder() - .width(entry->width) - .height(entry->height) - .levels(0xff) - .format(entry->srgb ? Texture::InternalFormat::SRGB8_A8 : Texture::InternalFormat::RGBA8) - .build(*mEngine); - asset->takeOwnership(entry->texture); - }; - for (auto& pair : mBufferTextureCache) createTexture(pair.second.get()); - for (auto& pair : mUriTextureCache) createTexture(pair.second.get()); - - // Bind the textures to material instances. - for (auto slot : asset->mTextureSlots) { - bindTextureToMaterial(slot); - } - - // Before creating jobs for PNG / JPEG decoding, we might need to return early. On single - // threaded systems, it is usually fine to create jobs because the job system will simply - // execute serially. However if the client requests async behavior, then we need to wait - // until subsequent calls to asyncUpdateLoad(). - if (!UTILS_HAS_THREADING && async) { - return true; - } - - JobSystem::Job* parent = js->createJob(); - - // Create a copy of the shared_ptr to the source data to prevent it from being freed during - // the texture decoding process. - FFilamentAsset::SourceHandle retainSourceAsset = asset->mSourceAsset; - - // Kick off jobs that decode texels from buffer pointers. - for (auto& pair : mBufferTextureCache) { - const uint8_t* sourceData = (const uint8_t*) pair.first; - TextureCacheEntry* entry = pair.second.get(); - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, sourceData] { - int width, height, comp; - entry->texels = stbi_load_from_memory(sourceData, entry->bufferSize, - &width, &height, &comp, 4); - }); - js->run(decode); - } - - // Kick off jobs that decode texels from URI strings. - for (auto& pair : mUriTextureCache) { - auto uri = pair.first; - TextureCacheEntry* entry = pair.second.get(); - - // First, check the user-supplied resource cache for this URI. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, sourceData, iter] { - int width, height, comp; - entry->texels = stbi_load_from_memory(sourceData, iter->second.size, &width, - &height, &comp, 4); - }); - js->run(decode); - continue; + if (Texture* texture = getOrCreateTexture(asset, slot)) { + asset->bindTexture(slot, texture); } - - // Otherwise load it from the file system if this platform supports it. - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - return false; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, fullpath] { - int width, height, comp; - entry->texels = stbi_load(fullpath.c_str(), &width, &height, &comp, 4); - }); - js->run(decode); - #endif } + // Non-threaded systems are required to use the asynchronous API. + assert_invariant(UTILS_HAS_THREADING || async); + if (async) { - mDecoderRootJob = js->runAndRetain(parent); return true; } - // Wait for decoding to finish. - js->runAndWait(parent); - - // Finally, upload texels to the GPU and generate mipmaps. - mCurrentAsset = asset; - uploadPendingTextures(); + for (const auto& iter : mTextureProviders) { + iter.second->waitForCompletion(); + iter.second->updateQueue(); + } return true; } @@ -971,8 +799,8 @@ void ResourceLoader::Impl::computeTangents(FFilamentAsset* asset) { } ResourceLoader::Impl::~Impl() { - if (mDecoderRootJob) { - mEngine->getJobSystem().waitAndRelease(mDecoderRootJob); + for (const auto& iter : mTextureProviders) { + iter.second->cancelDecoding(); } } @@ -1078,15 +906,18 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { for (cgltf_size slot = 0; slot < prim->attributes_count; slot++) { const cgltf_attribute& attr = prim->attributes[slot]; const cgltf_accessor* accessor = attr.data; - if (attr.type == cgltf_attribute_type_position && cgltf_num_components(accessor->type) >= posAttrSize) { + if (attr.type == cgltf_attribute_type_position && + cgltf_num_components(accessor->type) >= posAttrSize) { verts.resize(accessor->count * posAttrSize); cgltf_accessor_unpack_floats(accessor, &verts[0], accessor->count * posAttrSize); } - if (attr.type == cgltf_attribute_type_joints && cgltf_num_components(accessor->type) >= skinningAttrSize) { + if (attr.type == cgltf_attribute_type_joints && + cgltf_num_components(accessor->type) >= skinningAttrSize) { rawJoints.resize(accessor->count * skinningAttrSize); cgltf_accessor_unpack_floats(accessor, &rawJoints[0], accessor->count * skinningAttrSize); } - if (attr.type == cgltf_attribute_type_weights && cgltf_num_components(accessor->type) >= skinningAttrSize) { + if (attr.type == cgltf_attribute_type_weights && + cgltf_num_components(accessor->type) >= skinningAttrSize) { weights.resize(accessor->count * skinningAttrSize); cgltf_accessor_unpack_floats(accessor, &weights[0], accessor->count * skinningAttrSize); } @@ -1109,7 +940,8 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { if (!pImpl->mNormalizeSkinningWeights) { skinMatrix /= skinMatrix[3].w; } - float3 skinnedPoint = (point.x * skinMatrix[0] + point.y * skinMatrix[1] + point.z * skinMatrix[2] + skinMatrix[3]).xyz; + float3 skinnedPoint = (point.x * skinMatrix[0] + + point.y * skinMatrix[1] + point.z * skinMatrix[2] + skinMatrix[3]).xyz; aabb.min = min(aabb.min, skinnedPoint); aabb.max = max(aabb.max, skinnedPoint); } @@ -1117,7 +949,7 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { *result = aabb; }; - // Collect all mesh primitives that we wish to find bounds for. For each mesh primitive, we also + // Collect all mesh primitives that we wish to find bounds for. For each mesh primitive, we also // collect the skin it bound to (nullptr if not skinned) for bounds computation. std::vector> primitives; for (auto iter : nodeMap) { @@ -1126,7 +958,7 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { if (cgltfSkin) { // importSkins unpacked cgltfSkin into FFilamentInstance::SkinVector bijectively so that // the unpacked Skin can be retrieved given cgltfSkin index - int skinIndex = cgltfSkin - pImpl->cgltfSkinBaseAddress; + int skinIndex = cgltfSkin - pImpl->mSkinBaseAddress; skin = &asset->mSkins[skinIndex]; } const cgltf_mesh* cgltfMesh = iter.first->mesh; diff --git a/libs/gltfio/src/StbProvider.cpp b/libs/gltfio/src/StbProvider.cpp new file mode 100644 index 0000000000..8b1efce2ac --- /dev/null +++ b/libs/gltfio/src/StbProvider.cpp @@ -0,0 +1,261 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include + +#include +#include + +#include + +using namespace filament; +using namespace utils; + +using std::atomic; +using std::vector; +using std::unique_ptr; + +namespace gltfio { + +class StbProvider : public TextureProvider { +public: + StbProvider(Engine* engine); + ~StbProvider(); + + Texture* pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, uint64_t flags) final; + + Texture* popTexture() final; + void updateQueue() final; + void waitForCompletion() final; + void cancelDecoding() final; + const char* getPushMessage() const final; + const char* getPopMessage() const final; + size_t getPushedCount() const final { return mPushedCount; } + size_t getPoppedCount() const final { return mPoppedCount; } + size_t getDecodedCount() const final { return mDecodedCount; } + +private: + enum class TextureState { + DECODING, // Texture has been pushed, mipmap levels are not yet complete. + READY, // Mipmap levels are available but texture has not been popped yet. + POPPED, // Client has popped the texture from the queue. + }; + + struct TextureInfo { + Texture* texture; + TextureState state; + atomic decodedTexelsBaseMipmap; + vector sourceBuffer; + JobSystem::Job* decoderJob; + }; + + // Declare some sentinel values for the "decodedTexelsBaseMipmap" field. + // Note that the "state" field can be modified only on the foreground thread. + static const intptr_t DECODING_NOT_READY = 0x0; + static const intptr_t DECODING_ERROR = 0x1; + + void decodeSingleTexture(); + + size_t mPushedCount = 0; + size_t mPoppedCount = 0; + size_t mDecodedCount = 0; + vector > mTextures; + JobSystem::Job* mDecoderRootJob; + std::string mRecentPushMessage; + std::string mRecentPopMessage; + Engine* const mEngine; +}; + +Texture* StbProvider::pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, FlagBits flags) { + int width, height, numComponents; + if (!stbi_info_from_memory(data, byteCount, &width, &height, &numComponents)) { + mRecentPushMessage = std::string("Unable to parse texture: ") + stbi_failure_reason(); + return nullptr; + } + + using InternalFormat = Texture::InternalFormat; + const FlagBits sRGB = FlagBits(Flags::sRGB); + + Texture* texture = Texture::Builder() + .width(width) + .height(height) + .levels(0xff) + .format((flags & sRGB) ? InternalFormat::SRGB8_A8 : InternalFormat::RGBA8) + .build(*mEngine); + + if (texture == nullptr) { + mRecentPushMessage = "Unable to build Texture object."; + return nullptr; + } + + mRecentPushMessage.clear(); + TextureInfo* info = mTextures.emplace_back(new TextureInfo).get(); + ++mPushedCount; + + info->texture = texture; + info->state = TextureState::DECODING; + info->sourceBuffer.assign(data, data + byteCount); + info->decodedTexelsBaseMipmap.store(DECODING_NOT_READY); + + // On single threaded systems, it is usually fine to create jobs because the job system will + // simply execute serially. However in our case, we wish to amortize the decoder cost across + // several frames, so we instead use the updateQueue() method to perform decoding. + if constexpr (!UTILS_HAS_THREADING) { + info->decoderJob = nullptr; + return texture; + } + + JobSystem* js = &mEngine->getJobSystem(); + info->decoderJob = jobs::createJob(*js, mDecoderRootJob, [this, info] { + auto& source = info->sourceBuffer; + int width, height, comp; + + // Test asynchronous loading by uncommenting this line. + // std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 10000)); + + stbi_uc* texels = stbi_load_from_memory(source.data(), source.size(), + &width, &height, &comp, 4); + source.clear(); + source.shrink_to_fit(); + info->decodedTexelsBaseMipmap.store(texels ? intptr_t(texels) : DECODING_ERROR); + }); + + js->runAndRetain(info->decoderJob); + return texture; +} + +Texture* StbProvider::popTexture() { + // We don't bother shrinking the mTextures vector here, instead we periodically clean it up in + // the updateQueue method, since popTexture is typically called more frequently. Textures + // can become ready in non-deterministic order due to concurrency. + for (auto& texture : mTextures) { + if (texture->state == TextureState::READY) { + texture->state = TextureState::POPPED; + ++mPoppedCount; + const intptr_t ptr = texture->decodedTexelsBaseMipmap.load(); + if (ptr == DECODING_ERROR || ptr == DECODING_NOT_READY) { + mRecentPopMessage = "Texture is incomplete"; + } else { + mRecentPopMessage.clear(); + } + return texture->texture; + } + } + return nullptr; +} + +void StbProvider::updateQueue() { + if (!UTILS_HAS_THREADING) { + decodeSingleTexture(); + } + JobSystem* js = &mEngine->getJobSystem(); + for (auto& info : mTextures) { + if (info->state != TextureState::DECODING) { + continue; + } + Texture* texture = info->texture; + if (intptr_t data = info->decodedTexelsBaseMipmap.load()) { + if (info->decoderJob) { + js->waitAndRelease(info->decoderJob); + } + if (data == DECODING_ERROR) { + info->state = TextureState::READY; + ++mDecodedCount; + continue; + } + Texture::PixelBufferDescriptor pbd((uint8_t*) data, + texture->getWidth() * texture->getHeight() * 4, Texture::Format::RGBA, + Texture::Type::UBYTE, [](void* mem, size_t, void*) { stbi_image_free(mem); }); + texture->setImage(*mEngine, 0, std::move(pbd)); + + // Call generateMipmaps unconditionally to fulfill the promise of the TextureProvider + // interface. Providers of hierarchical images (e.g. KTX) call this only if needed. + texture->generateMipmaps(*mEngine); + + info->state = TextureState::READY; + ++mDecodedCount; + } + } + + // Here we periodically clean up the "queue" (which is really just a vector) by removing unused + // items from the front. This might ignore a popped texture that occurs in the middle of the + // vector, but that's okay, it will be cleaned up eventually. + decltype(mTextures)::iterator last = mTextures.begin(); + while (last != mTextures.end() && (*last)->state == TextureState::POPPED) ++last; + mTextures.erase(mTextures.begin(), last); +} + +void StbProvider::waitForCompletion() { + JobSystem& js = mEngine->getJobSystem(); + for (auto& info : mTextures) { + if (info->decoderJob) { + js.waitAndRelease(info->decoderJob); + } + } +} + +void StbProvider::cancelDecoding() { + // TODO: Currently, StbProvider runs jobs eagerly and JobSystem does not allow cancellation of + // in-flight jobs. We should consider throttling the number of simultaneous decoder jobs, which + // would allow for actual cancellation. + waitForCompletion(); +} + +const char* StbProvider::getPushMessage() const { + return mRecentPushMessage.empty() ? nullptr : mRecentPushMessage.c_str(); +} + +const char* StbProvider::getPopMessage() const { + return mRecentPopMessage.empty() ? nullptr : mRecentPopMessage.c_str(); +} + +void StbProvider::decodeSingleTexture() { + assert_invariant(!UTILS_HAS_THREADING); + for (auto& info : mTextures) { + if (info->state == TextureState::DECODING) { + auto& source = info->sourceBuffer; + int width, height, comp; + stbi_uc* texels = stbi_load_from_memory(source.data(), source.size(), + &width, &height, &comp, 4); + source.clear(); + source.shrink_to_fit(); + info->decodedTexelsBaseMipmap.store(texels ? intptr_t(texels) : DECODING_ERROR); + break; + } + } +} + +StbProvider::StbProvider(Engine* engine) : mEngine(engine) { + mDecoderRootJob = mEngine->getJobSystem().createJob(); +} + +StbProvider::~StbProvider() { + cancelDecoding(); + mEngine->getJobSystem().release(mDecoderRootJob); +} + +TextureProvider* createStbProvider(Engine* engine) { + return new StbProvider(engine); +} + +} // namespace gltfio diff --git a/libs/viewer/include/viewer/AutomationEngine.h b/libs/viewer/include/viewer/AutomationEngine.h index ac8b98f721..5617d82b6f 100644 --- a/libs/viewer/include/viewer/AutomationEngine.h +++ b/libs/viewer/include/viewer/AutomationEngine.h @@ -40,7 +40,7 @@ namespace viewer { * * When executing a test, clients should call tick() after each frame is rendered, which gives * automation an opportunity to push settings to Filament, increment the current test index (if - * enough time has elapsed), and request an asychronous screenshot. + * enough time has elapsed), and request an asynchronous screenshot. * * The time to sleep between tests is configurable and can be set to zero. Automation also waits a * specified minimum number of frames between tests. diff --git a/samples/gltf_viewer.cpp b/samples/gltf_viewer.cpp index a2cae36747..92cb656651 100644 --- a/samples/gltf_viewer.cpp +++ b/samples/gltf_viewer.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,7 @@ struct App { MaterialSource materialSource = GENERATE_SHADERS; gltfio::ResourceLoader* resourceLoader = nullptr; + gltfio::TextureProvider* decoder = nullptr; bool recomputeAabb = false; bool ignoreBindTransform = false; @@ -423,6 +425,9 @@ int main(int argc, char** argv) { configuration.normalizeSkinningWeights = true; if (!app.resourceLoader) { app.resourceLoader = new gltfio::ResourceLoader(configuration); + app.decoder = createStbProvider(app.engine); + app.resourceLoader->addTextureProvider("image/png", app.decoder); + app.resourceLoader->addTextureProvider("image/jpeg", app.decoder); } app.resourceLoader->asyncBeginLoad(app.asset); app.asset->releaseSourceData(); @@ -644,6 +649,7 @@ int main(int argc, char** argv) { delete app.viewer; delete app.materials; delete app.names; + delete app.decoder; AssetLoader::destroy(&app.assetLoader); }; diff --git a/third_party/stb/stb_image.cpp b/third_party/stb/stb_image.cpp new file mode 100644 index 0000000000..3ee30bdfb5 --- /dev/null +++ b/third_party/stb/stb_image.cpp @@ -0,0 +1,9 @@ +#define STBI_NO_BMP +#define STBI_NO_PSD +#define STBI_NO_TGA +#define STBI_NO_GIF +#define STBI_NO_PIC +#define STBI_NO_PNM + +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" diff --git a/third_party/stb/tnt/CMakeLists.txt b/third_party/stb/tnt/CMakeLists.txt index b952728584..330ff4f694 100644 --- a/third_party/stb/tnt/CMakeLists.txt +++ b/third_party/stb/tnt/CMakeLists.txt @@ -2,12 +2,15 @@ cmake_minimum_required(VERSION 3.19) project(stb) set(TARGET stb) -set(OUR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -set(PUBLIC_HDR_DIR ${OUR_DIR}) -add_library(${TARGET} INTERFACE) +set(PUBLIC_HDR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_include_directories(${TARGET} INTERFACE ${PUBLIC_HDR_DIR}) +set(PUBLIC_HDRS ../stb_image.h) -file(GLOB_RECURSE PUBLIC_HDRS ${PUBLIC_HDR_DIR}/*.h) -target_sources(${TARGET} INTERFACE ${PUBLIC_HDRS}) +set(SRCS ../stb_image.cpp) + +add_library(${TARGET} STATIC ${PUBLIC_HDRS} ${SRCS}) + +target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) + +install(TARGETS stb ARCHIVE DESTINATION lib/${DIST_DIR}) diff --git a/web/filament-js/extensions.js b/web/filament-js/extensions.js index 519095e4fa..7e0c58f0ce 100644 --- a/web/filament-js/extensions.js +++ b/web/filament-js/extensions.js @@ -643,6 +643,12 @@ Filament.loadClassExtensions = function() { config.normalizeSkinningWeights, config.recomputeBoundingBoxes, config.ignoreBindTransform); + + const stbProvider = new Filament.gltfio$StbProvider(engine); + + resourceLoader.addTextureProvider("image/jpeg", stbProvider); + resourceLoader.addTextureProvider("image/png", stbProvider); + const onComplete = () => { resourceLoader.asyncBeginLoad(asset); @@ -657,6 +663,7 @@ Filament.loadClassExtensions = function() { if (progress >= 1) { clearInterval(timer); resourceLoader.delete(); + stbProvider.delete(); onDone(); } }, interval); diff --git a/web/filament-js/jsbindings.cpp b/web/filament-js/jsbindings.cpp index c81b23da4f..1f1ce30ca2 100644 --- a/web/filament-js/jsbindings.cpp +++ b/web/filament-js/jsbindings.cpp @@ -64,9 +64,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -80,6 +80,8 @@ #include #include +#include + #include #include @@ -1886,18 +1888,28 @@ class_("gltfio$FilamentInstance") .function("applyMaterialVariant", &FilamentInstance::applyMaterialVariant) .function("getAnimator", &FilamentInstance::getAnimator, allow_raw_pointers()); -// This little wrapper exists to get around RTTI requirements in embind. +// These little wrappers exist to get around RTTI requirements in embind. + struct UbershaderLoader { MaterialProvider* provider; void destroyMaterials() { provider->destroyMaterials(); } }; +struct StbProvider { + TextureProvider* provider; +}; + class_("gltfio$UbershaderLoader") .constructor(EMBIND_LAMBDA(UbershaderLoader, (Engine* engine), { return UbershaderLoader { createUbershaderLoader(engine) }; })) .function("destroyMaterials", &UbershaderLoader::destroyMaterials); +class_("gltfio$StbProvider") + .constructor(EMBIND_LAMBDA(StbProvider, (Engine* engine), { + return StbProvider { createStbProvider(engine) }; + })); + class_("gltfio$AssetLoader") .constructor(EMBIND_LAMBDA(AssetLoader*, (Engine* engine, UbershaderLoader materials), { @@ -1959,6 +1971,11 @@ class_("gltfio$ResourceLoader") self->addResourceData(url.c_str(), std::move(*buffer.bd)); }), allow_raw_pointers()) + .function("addTextureProvider", EMBIND_LAMBDA(void, (ResourceLoader* self, std::string mime, + StbProvider provider), { + self->addTextureProvider(mime.c_str(), provider.provider); + }), allow_raw_pointers()) + .function("hasResourceData", EMBIND_LAMBDA(bool, (ResourceLoader* self, std::string url), { return self->hasResourceData(url.c_str()); }), allow_raw_pointers())