From f847232296256c03a089823ddc36cbc7d29f0f67 Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Fri, 15 Apr 2022 04:05:04 -0700 Subject: [PATCH] gltfio: introduce TextureProvider API. This allows clients to provide their own asynchronous texture decoders for various mime-typed images. This is a plug-in component for gltfio, in some ways similar to MaterialProvider. There are two motivations for this: to decouple gltfio from STB and to make it easier to integrate support for BasisU textures. This also has the side effect of simplifying ResourceLoader, since the texture decoding jobs have been moved out. As part of this work, I made the "stb" CMake target into a traditional static library. Previously we had several files called `Image.cpp` whose sole purpose was to enable STB_IMAGE_IMPLEMENTATION. --- RELEASE_NOTES.md | 1 + .../filament/utils/AutomationEngine.java | 2 +- android/gltfio-android/CMakeLists.txt | 11 +- .../src/main/cpp/ResourceLoader.cpp | 25 + .../filament/gltfio/ResourceLoader.java | 12 + .../gltf-viewer/gltf-viewer/FILModelView.mm | 6 + libs/filamentapp/CMakeLists.txt | 1 - libs/filamentapp/src/Image.cpp | 25 - libs/gltfio/CMakeLists.txt | 8 +- libs/gltfio/include/gltfio/AssetLoader.h | 7 +- libs/gltfio/include/gltfio/Image.h | 32 -- libs/gltfio/include/gltfio/ResourceLoader.h | 8 + libs/gltfio/include/gltfio/TextureProvider.h | 174 +++++++ libs/gltfio/src/DependencyGraph.cpp | 4 +- libs/gltfio/src/DependencyGraph.h | 25 +- libs/gltfio/src/Image.cpp | 19 - libs/gltfio/src/ResourceLoader.cpp | 488 ++++++------------ libs/gltfio/src/StbProvider.cpp | 261 ++++++++++ libs/viewer/include/viewer/AutomationEngine.h | 2 +- samples/gltf_viewer.cpp | 6 + third_party/stb/stb_image.cpp | 9 + third_party/stb/tnt/CMakeLists.txt | 15 +- web/filament-js/extensions.js | 7 + web/filament-js/jsbindings.cpp | 21 +- 24 files changed, 736 insertions(+), 433 deletions(-) delete mode 100644 libs/filamentapp/src/Image.cpp delete mode 100644 libs/gltfio/include/gltfio/Image.h create mode 100644 libs/gltfio/include/gltfio/TextureProvider.h delete mode 100644 libs/gltfio/src/Image.cpp create mode 100644 libs/gltfio/src/StbProvider.cpp create mode 100644 third_party/stb/stb_image.cpp diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4026192c5b..37b23ce26b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -11,6 +11,7 @@ A new header is inserted each time a *tag* is created. - Removed KTX1 compression support from mipgen. [⚠️ **API Change**]. - Added support for KTX2 to mipgen. - gltfio: Added resetBoneMatrices() method. +- gltfio: Introduced `TextureProvider` interface. [⚠️ **API Change**]. ## v1.21.2 diff --git a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java index 1fa0feaaed..4383c2161d 100644 --- a/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java +++ b/android/filament-utils-android/src/main/java/com/google/android/filament/utils/AutomationEngine.java @@ -40,7 +40,7 @@ import com.google.android.filament.Renderer; * * When executing a test, clients should call tick() after each frame is rendered, which provides an * opportunity to push settings to Filament, increment the current test index (if enough time has - * elapsed), and request an asychronous screenshot. + * elapsed), and request an asynchronous screenshot. * * The time to sleep between tests is configurable and can be set to zero. Automation also waits a * specified minimum number of frames between tests. diff --git a/android/gltfio-android/CMakeLists.txt b/android/gltfio-android/CMakeLists.txt index b7a8c4471d..3a5b4f5f4f 100644 --- a/android/gltfio-android/CMakeLists.txt +++ b/android/gltfio-android/CMakeLists.txt @@ -11,6 +11,10 @@ add_library(dracodec STATIC IMPORTED) set_target_properties(dracodec PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libdracodec.a) +add_library(stb STATIC IMPORTED) +set_target_properties(stb PROPERTIES IMPORTED_LOCATION + ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libstb.a) + add_library(utils STATIC IMPORTED) set_target_properties(utils PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libutils.a) @@ -32,6 +36,7 @@ set(GLTFIO_SRCS ${GLTFIO_DIR}/include/gltfio/ResourceLoader.h ${GLTFIO_DIR}/include/gltfio/FilamentAsset.h ${GLTFIO_DIR}/include/gltfio/FilamentInstance.h + ${GLTFIO_DIR}/include/gltfio/TextureProvider.h ${GLTFIO_DIR}/include/gltfio/math.h ${GLTFIO_DIR}/src/Animator.cpp @@ -47,13 +52,13 @@ set(GLTFIO_SRCS ${GLTFIO_DIR}/src/GltfEnums.h ${GLTFIO_DIR}/src/MaterialProvider.cpp ${GLTFIO_DIR}/src/ResourceLoader.cpp + ${GLTFIO_DIR}/src/StbProvider.cpp ${GLTFIO_DIR}/src/TangentsJob.h ${GLTFIO_DIR}/src/TangentsJob.cpp ${GLTFIO_DIR}/src/UbershaderLoader.cpp ${GLTFIO_DIR}/src/Wireframe.cpp ${GLTFIO_DIR}/src/Wireframe.h ${GLTFIO_DIR}/src/upcast.h - ${GLTFIO_DIR}/src/Image.cpp src/main/cpp/Animator.cpp src/main/cpp/AssetLoader.cpp @@ -90,9 +95,9 @@ set_target_properties(gltfio-jni PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_ if(GLTFIO_LITE) target_compile_definitions(gltfio-jni PUBLIC GLTFIO_LITE=1) - target_link_libraries(gltfio-jni filament-jni utils log gltfio_resources_lite) + target_link_libraries(gltfio-jni filament-jni utils log stb gltfio_resources_lite) else() - target_link_libraries(gltfio-jni filament-jni utils log gltfio_resources) + target_link_libraries(gltfio-jni filament-jni utils log stb gltfio_resources) # Enable Draco in the non-lite variant of gltfio. target_link_libraries(gltfio-jni dracodec) diff --git a/android/gltfio-android/src/main/cpp/ResourceLoader.cpp b/android/gltfio-android/src/main/cpp/ResourceLoader.cpp index c16ba604ed..d1a48a9750 100644 --- a/android/gltfio-android/src/main/cpp/ResourceLoader.cpp +++ b/android/gltfio-android/src/main/cpp/ResourceLoader.cpp @@ -19,6 +19,7 @@ #include #include +#include #include @@ -114,3 +115,27 @@ Java_com_google_android_filament_gltfio_ResourceLoader_nAsyncCancelLoad(JNIEnv*, ResourceLoader* loader = (ResourceLoader*) nativeLoader; loader->asyncCancelLoad(); } + +extern "C" JNIEXPORT jlong JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nCreateTextureProvider(JNIEnv*, jclass, + jlong nativeEngine) { + Engine* engine = (Engine*) nativeEngine; + return (jlong) createStbProvider(engine); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nDestroyTextureProvider(JNIEnv*, jclass, + jlong nativeProvider) { + TextureProvider* provider = (TextureProvider*) nativeProvider; + delete provider; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_android_filament_gltfio_ResourceLoader_nAddTextureProvider(JNIEnv* env, jclass, + jlong nativeLoader, jstring url, jlong nativeProvider) { + ResourceLoader* loader = (ResourceLoader*) nativeLoader; + TextureProvider* provider = (TextureProvider*) nativeProvider; + const char* cstring = env->GetStringUTFChars(url, nullptr); + loader->addTextureProvider(cstring, provider); + env->ReleaseStringUTFChars(url, cstring); +} diff --git a/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java b/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java index 976a752caf..217909c2ff 100644 --- a/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java +++ b/android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java @@ -35,6 +35,7 @@ import java.nio.Buffer; */ public class ResourceLoader { private final long mNativeObject; + private final long mNativeProvider; /** * Constructs a resource loader tied to the given Filament engine. @@ -46,6 +47,9 @@ public class ResourceLoader { public ResourceLoader(@NonNull Engine engine) { long nativeEngine = engine.getNativeObject(); mNativeObject = nCreateResourceLoader(nativeEngine, false, false, false); + mNativeProvider = nCreateTextureProvider(nativeEngine); + nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider); + nAddTextureProvider(mNativeObject, "image/png", mNativeProvider); } /** @@ -63,6 +67,9 @@ public class ResourceLoader { long nativeEngine = engine.getNativeObject(); mNativeObject = nCreateResourceLoader(nativeEngine, normalizeSkinningWeights, recomputeBoundingBoxes, ignoreBindTransform); + mNativeProvider = nCreateTextureProvider(nativeEngine); + nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider); + nAddTextureProvider(mNativeObject, "image/png", mNativeProvider); } /** @@ -70,6 +77,7 @@ public class ResourceLoader { */ public void destroy() { nDestroyResourceLoader(mNativeObject); + nDestroyTextureProvider(mNativeObject); } /** @@ -178,4 +186,8 @@ public class ResourceLoader { private static native float nAsyncGetLoadProgress(long nativeLoader); private static native void nAsyncUpdateLoad(long nativeLoader); private static native void nAsyncCancelLoad(long nativeLoader); + + private static native long nCreateTextureProvider(long nativeEngine); + private static native void nAddTextureProvider(long nativeLoader, String url, long nativeProvider); + private static native void nDestroyTextureProvider(long nativeProvider); } diff --git a/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm b/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm index 8c2e98b547..75f5e2de98 100644 --- a/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm +++ b/ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -78,6 +79,7 @@ const float kSensitivity = 100.0f; ResourceLoader* _resourceLoader; Manipulator* _manipulator; + TextureProvider* _decoder; FilamentAsset* _asset; @@ -129,6 +131,9 @@ const float kSensitivity = 100.0f; _assetLoader = AssetLoader::create({_engine, _materialProvider, ncm, &em}); _resourceLoader = new ResourceLoader( {.engine = _engine, .normalizeSkinningWeights = true, .recomputeBoundingBoxes = false}); + _decoder = createStbProvider(_engine); + _resourceLoader->addTextureProvider("image/png", _decoder); + _resourceLoader->addTextureProvider("image/jpeg", _decoder); _manipulator = Manipulator::Builder().orbitHomePosition(0.0f, 0.0f, 4.0f).build(Mode::ORBIT); @@ -247,6 +252,7 @@ const float kSensitivity = 100.0f; [self destroyModel]; delete _manipulator; + delete _decoder; _materialProvider->destroyMaterials(); delete _materialProvider; diff --git a/libs/filamentapp/CMakeLists.txt b/libs/filamentapp/CMakeLists.txt index 7414278ce9..5855d0d032 100644 --- a/libs/filamentapp/CMakeLists.txt +++ b/libs/filamentapp/CMakeLists.txt @@ -33,7 +33,6 @@ set(SRCS src/FilamentApp.cpp src/IBL.cpp src/IcoSphere.cpp - src/Image.cpp src/MeshAssimp.cpp src/Sphere.cpp ) diff --git a/libs/filamentapp/src/Image.cpp b/libs/filamentapp/src/Image.cpp deleted file mode 100644 index 38a8c06c2b..0000000000 --- a/libs/filamentapp/src/Image.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_TGA -#define STBI_NO_GIF -#define STBI_NO_PIC -#define STBI_NO_PNM - -#define STB_IMAGE_IMPLEMENTATION -#include diff --git a/libs/gltfio/CMakeLists.txt b/libs/gltfio/CMakeLists.txt index 1ee2113135..9fe414a458 100644 --- a/libs/gltfio/CMakeLists.txt +++ b/libs/gltfio/CMakeLists.txt @@ -14,6 +14,7 @@ set(PUBLIC_HDRS include/gltfio/ResourceLoader.h include/gltfio/FilamentAsset.h include/gltfio/FilamentInstance.h + include/gltfio/TextureProvider.h include/gltfio/math.h ) @@ -31,13 +32,13 @@ set(SRCS src/GltfEnums.h src/MaterialProvider.cpp src/ResourceLoader.cpp + src/StbProvider.cpp src/TangentsJob.h src/TangentsJob.cpp src/UbershaderLoader.cpp src/Wireframe.cpp src/Wireframe.h src/upcast.h - src/Image.cpp ) # ================================================================================================== @@ -196,10 +197,7 @@ if (NOT WEBGL AND NOT ANDROID AND NOT IOS) # Installation # ================================================================================================== install(TARGETS ${TARGET} gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) - # The Image.h header is only needed within Filament and does not need to be distributed. - install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio - DESTINATION include - PATTERN "Image.h" EXCLUDE) + install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio DESTINATION include) else() install(TARGETS gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) diff --git a/libs/gltfio/include/gltfio/AssetLoader.h b/libs/gltfio/include/gltfio/AssetLoader.h index 81feaeb23d..90491c9852 100644 --- a/libs/gltfio/include/gltfio/AssetLoader.h +++ b/libs/gltfio/include/gltfio/AssetLoader.h @@ -84,6 +84,7 @@ struct AssetConfiguration { * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * auto engine = Engine::create(); * auto materials = createMaterialGenerator(engine); + * auto decoder = createStbProvider(engine); * auto loader = AssetLoader::create({engine, materials}); * * // Parse the glTF content and create Filament entities. @@ -92,7 +93,10 @@ struct AssetConfiguration { * content.clear(); * * // Load buffers and textures from disk. - * ResourceLoader({engine, ".", true}).loadResources(asset); + * ResourceLoader resourceLoader({engine, ".", true}); + * resourceLoader.addTextureProvider("image/png", decoder) + * resourceLoader.addTextureProvider("image/jpeg", decoder) + * resourceLoader.loadResources(asset); * * // Free the glTF hierarchy as it is no longer needed. * asset->releaseSourceData(); @@ -114,6 +118,7 @@ struct AssetConfiguration { * loader->destroyAsset(asset); * materials->destroyMaterials(); * delete materials; + * delete decoder; * AssetLoader::destroy(&loader); * Engine::destroy(&engine); * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/libs/gltfio/include/gltfio/Image.h b/libs/gltfio/include/gltfio/Image.h deleted file mode 100644 index c7047311d2..0000000000 --- a/libs/gltfio/include/gltfio/Image.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// gltfio supports PNG and JPEG, disable all other formats. -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_TGA -#define STBI_NO_GIF -#define STBI_NO_HDR -#define STBI_NO_PIC -#define STBI_NO_PNM - -// For emscripten and Android builds, we never load from the file -// system, so we-opt out of the stdio functionality in stb. -#if defined(__EMSCRIPTEN__) || defined(__ANDROID__) -#define STBI_NO_STDIO -#endif - -#include diff --git a/libs/gltfio/include/gltfio/ResourceLoader.h b/libs/gltfio/include/gltfio/ResourceLoader.h index a92a4f8517..9de8bc2f25 100644 --- a/libs/gltfio/include/gltfio/ResourceLoader.h +++ b/libs/gltfio/include/gltfio/ResourceLoader.h @@ -31,6 +31,7 @@ namespace gltfio { struct FFilamentAsset; class AssetPool; +class TextureProvider; /** * \struct ResourceConfiguration ResourceLoader.h gltfio/ResourceLoader.h @@ -94,6 +95,13 @@ public: */ void addResourceData(const char* uri, BufferDescriptor&& buffer); + /** + * Register a plugin that can consume PNG / JPEG content and produce filament::Texture objects. + * + * Destruction of the given provider is the client's responsibility. + */ + void addTextureProvider(const char* mimeType, TextureProvider* provider); + /** * Checks if the given resource has already been added to the URI cache. */ diff --git a/libs/gltfio/include/gltfio/TextureProvider.h b/libs/gltfio/include/gltfio/TextureProvider.h new file mode 100644 index 0000000000..3bdbba0d80 --- /dev/null +++ b/libs/gltfio/include/gltfio/TextureProvider.h @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GLTFIO_TEXTUREPROVIDER_H +#define GLTFIO_TEXTUREPROVIDER_H + +#include +#include + +#include + +namespace filament { + class Engine; + class Texture; +} + +namespace gltfio { + +/** + * TextureProvider is an interface that allows clients to implement their own texture decoding + * facility for JPEG, PNG, or KTX2 content. It constructs Filament Texture objects synchronously, + * but populates their miplevels asynchronously. + * + * gltfio calls all public methods from the foreground thread, i.e. the thread that the Filament + * engine was created with. However the implementation may create 0 or more background threads to + * perform decoding work. + * + * The following pseudocode illustrates how this interface could be used, but in practice the only + * client is the gltfio ResourceLoader. + * + * filament::Engine* engine = ...; + * TextureProvider* provider = createStbProvider(engine); + * + * for (auto filename : textureFiles) { + * std::vector buf = readEntireFile(filename); + * Texture* texture = provider->pushTexture(buf.data(), buf.size(), "image/png", 0); + * if (texture == nullptr) { puts(provider->getPushMessage()); exit(1); } + * } + * + * // At this point, the returned textures can be bound to material instances, but none of their + * // miplevel images have been populated yet. + * + * while (provider->getPoppedCount() < provider->getPushedCount()) { + * sleep(200); + * + * // The following call gives the provider an opportunity to reap the results of any + * // background decoder work that has been completed (e.g. by calling Texture::setImage). + * provider->updateQueue(); + * + * // Check for textures that now have all their miplevels initialized. + * while (Texture* texture = provider->popTexture()) { + * printf("%p has all its miplevels ready.\n", texture); + * } + * } + * + * delete provider; + */ +class UTILS_PUBLIC TextureProvider { +public: + using Texture = filament::Texture; + using FlagBits = uint64_t; + + enum class Flags { + sRGB = 1 << 0, + }; + + /** + * Creates a Filament texture and pushes it to the asynchronous decoding queue. + * + * If construction fails, nothing is pushed to the queue and null is returned. The failure + * reason can be obtained with getPushMessage(). The given buffer pointer is not held, so the + * caller can free it immediately. It is also the caller's responsibility to free the returned + * Texture object, but it is only safe to do so after it has been popped from the queue. + */ + virtual Texture* pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, FlagBits flags) = 0; + + /** + * Checks if any texture is ready to be removed from the asynchronous decoding queue, and if so + * pops it off. + * + * Unless an error or cancellation occurred during the decoding process, the returned texture + * should have all its miplevels populated. If the texture is not complete, the reason can be + * obtained with getPopMessage(). + * + * Due to concurrency, textures are not necessarily popped off in the same order they were + * pushed. Returns null if there are no textures that are ready to be popped. + */ + virtual Texture* popTexture() = 0; + + /** + * Polls textures in the queue and uploads mipmap images if any have emerged from the decoder. + * + * This gives the provider an opportunity to call Texture::setImage() on the foreground thread. + * If needed, it can also call Texture::generateMipmaps() here. + * + * Items in the decoding queue can become "poppable" only during this call. + */ + virtual void updateQueue() = 0; + + /** + * Returns a failure message for the most recent call to pushTexture(), or null for success. + * + * Note that this method does not pertain to the decoding process. If decoding fails, clients to + * can pop the incomplete texture off the queue and obtain a failure message using the + * getPopFailure() method. + * + * The returned string is owned by the provider and becomes invalid after the next call to + * pushTexture(). + */ + virtual const char* getPushMessage() const = 0; + + /** + * Returns a failure message for the most recent call to popTexture(), or null for success. + * + * If the most recent call to popTexture() returned null, then no error occurred and this + * returns null. If the most recent call to popTexture() returned a "complete" texture (i.e. + * all miplevels present), then this returns null. This returns non-null only if an error or + * cancellation occurred while decoding the popped texture. + * + * The returned string is owned by the provider and becomes invalid after the next call to + * popTexture(). + */ + virtual const char* getPopMessage() const = 0; + + /** + * Waits for all outstanding decoding jobs to complete. + * + * Clients should call updateQueue() afterwards if they wish to update the push / pop queue. + */ + virtual void waitForCompletion() = 0; + + /** + * Cancels all not-yet-started decoding jobs and waits for all other jobs to complete. + * + * Jobs that have already started cannot be canceled. Textures whose decoding process has + * been cancelled will be made poppable on the subsequent call to updateQueue(). + */ + virtual void cancelDecoding() = 0; + + /** Total number of successful push calls since the provider was created. */ + virtual size_t getPushedCount() const = 0; + + /** Total number of successful pop calls since the provider was created. */ + virtual size_t getPoppedCount() const = 0; + + /** Total number of textures that have become ready-to-pop since the provider was created. */ + virtual size_t getDecodedCount() const = 0; + + virtual ~TextureProvider() = default; +}; + +/** + * Creates a simple decoder based on stb_image that can handle "image/png" and "image/jpeg". + * This works only if your build configuration includes STB. + */ +TextureProvider* createStbProvider(filament::Engine* engine); + +} // namespace gltfio + +#endif // GLTFIO_TEXTUREPROVIDER_H diff --git a/libs/gltfio/src/DependencyGraph.cpp b/libs/gltfio/src/DependencyGraph.cpp index 24812b03c3..9a92dbf7c6 100644 --- a/libs/gltfio/src/DependencyGraph.cpp +++ b/libs/gltfio/src/DependencyGraph.cpp @@ -75,7 +75,7 @@ void DependencyGraph::refinalize() { } void DependencyGraph::addEdge(Texture* texture, MaterialInstance* material, const char* parameter) { - assert(mFinalized); + assert(texture && !mFinalized); mTextureToMaterial[texture].insert(material); mMaterialToTexture.at(material).params.at(parameter) = getStatus(texture); } @@ -86,6 +86,7 @@ void DependencyGraph::checkReadiness(Material* material) { // Check this material's texture parameters, there are 5 in the worst case. bool materialIsReady = true; for (auto pair : status.params) { + assert(pair.second && "Parameter-to-Texture edge is missing."); if (!pair.second->ready) { materialIsReady = false; break; @@ -125,6 +126,7 @@ void DependencyGraph::markAsReady(MaterialInstance* material) { } DependencyGraph::TextureNode* DependencyGraph::getStatus(Texture* texture) { + assert(texture); auto iter = mTextureNodes.find(texture); if (iter == mTextureNodes.end()) { TextureNode* status = (mTextureNodes[texture] = std::make_unique()).get(); diff --git a/libs/gltfio/src/DependencyGraph.h b/libs/gltfio/src/DependencyGraph.h index 854571c2f4..b391cbc109 100644 --- a/libs/gltfio/src/DependencyGraph.h +++ b/libs/gltfio/src/DependencyGraph.h @@ -63,6 +63,7 @@ public: using Entity = utils::Entity; // Pops up to "count" ready-to-render entities off the queue. + // // If "result" is non-null, returns the number of written items. // If "result" is null, returns the number of available entities. size_t popRenderables(Entity* result, size_t count) noexcept; @@ -70,19 +71,27 @@ public: // These are called during the initial asset loader phase. void addEdge(Entity entity, Material* material); void addEdge(Material* material, const char* parameter); + void addEdge(filament::Texture* texture, Material* material, const char* parameter); - // This is called at the end of the initial asset loading phase. - // Makes a guarantee that no new material nodes or parameter nodes will be added to the graph. + // Marks the end of synchronous asset loading. + // + // At this point, the graph enters a finalized state and all non-textured entities are + // immediately marked as "ready". However textures are not yet fully decoded. + // + // After finalization, the only nodes that can be added to the graph are entities. void finalize(); - // This can be called after finalization to allow for dynamic addition of entities. - // It is slower than finalize() because it checks the readiness of existing materials. - void refinalize(); - - // These are called after textures have created and decoded. - void addEdge(filament::Texture* texture, Material* material, const char* parameter); + // Marks the given texture as being fully decoded, with all miplevels initialized. + // + // This can only be called on a finalized graph. void markAsReady(filament::Texture* texture); + // Re-checks the readiness of all entities after finalization. + // + // This exists only to support dynamic instancing. It is slower than finalize() because it + // checks the readiness of existing materials. + void refinalize(); + private: struct TextureNode { filament::Texture* texture; diff --git a/libs/gltfio/src/Image.cpp b/libs/gltfio/src/Image.cpp deleted file mode 100644 index 815649b5ec..0000000000 --- a/libs/gltfio/src/Image.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define STB_IMAGE_IMPLEMENTATION - -#include diff --git a/libs/gltfio/src/ResourceLoader.cpp b/libs/gltfio/src/ResourceLoader.cpp index 5fbe74e6cf..47737287ae 100644 --- a/libs/gltfio/src/ResourceLoader.cpp +++ b/libs/gltfio/src/ResourceLoader.cpp @@ -15,7 +15,7 @@ */ #include -#include +#include #include "GltfEnums.h" #include "FFilamentAsset.h" @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -45,12 +46,12 @@ #include #include +#include #if defined(__EMSCRIPTEN__) || defined(__ANDROID__) || defined(IOS) #define USE_FILESYSTEM 0 #else #define USE_FILESYSTEM 1 -#include #endif using namespace filament; @@ -62,25 +63,13 @@ using filament::geometry::ComponentType; static const auto FREE_CALLBACK = [](void* mem, size_t, void*) { free(mem); }; -namespace { - struct TextureCacheEntry { - Texture* texture; - std::atomic texels; - uint32_t bufferSize; - int width; - int height; - int numComponents; - bool srgb; - bool completed; - }; - - using BufferTextureCache = tsl::robin_map>; - using UriTextureCache = tsl::robin_map>; - using UriDataCache = tsl::robin_map; -} - namespace gltfio { +using BufferTextureCache = tsl::robin_map; +using FilepathTextureCache = tsl::robin_map; +using UriDataCache = tsl::robin_map; +using TextureProviderList = tsl::robin_map; + struct ResourceLoader::Impl { Impl(const ResourceConfiguration& config) { mGltfPath = std::string(config.gltfPath ? config.gltfPath : ""); @@ -96,33 +85,27 @@ struct ResourceLoader::Impl { bool mIgnoreBindTransform; std::string mGltfPath; - // This is used to calculate skinIndex when updateBoundingBoxes, so that the correspondency between + // This is used to calculate skinIndex when updateBoundingBoxes, so that the mapping between // cgltf_node* and FFilamentInstance::Skin can be retrieved. This pointer doesn't need to be freed. - cgltf_skin* cgltfSkinBaseAddress; + cgltf_skin* mSkinBaseAddress; // User-provided resource data with URI string keys, populated with addResourceData(). // This is used on platforms without traditional file systems, such as Android, iOS, and WebGL. UriDataCache mUriDataCache; - // The two texture caches are populated while textures are being decoded, and they are no longer - // used after all textures have been finalized. Since multiple glTF textures might be loaded - // from a single URI or buffer pointer, these caches prevent needless re-decoding. There are - // two caches: one for URI-based textures and one for buffer-based textures. + // User-provided mapping from mime types to texture providers. + TextureProviderList mTextureProviders; + + // Avoid duplicated Texture objects via caches with two key types: buffer pointers and strings. BufferTextureCache mBufferTextureCache; - UriTextureCache mUriTextureCache; - int mNumDecoderTasks; - int mNumDecoderTasksFinished; - JobSystem::Job* mDecoderRootJob = nullptr; - FFilamentAsset* mCurrentAsset = nullptr; + FilepathTextureCache mFilepathTextureCache; + + FFilamentAsset* mAsyncAsset = nullptr; void computeTangents(FFilamentAsset* asset); - bool createTextures(bool async); + bool createTextures(FFilamentAsset* asset, bool async); void cancelTextureDecoding(); - void addTextureCacheEntry(const TextureSlot& tb); - void bindTextureToMaterial(const TextureSlot& tb); - void decodeSingleTexture(); - void uploadPendingTextures(); - void releasePendingTextures(); + Texture* getOrCreateTexture(FFilamentAsset* asset, const TextureSlot& tb); ~Impl(); }; @@ -393,7 +376,8 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { if (strncmp(uri, "data:", 5) == 0) { const char* comma = strchr(uri, ','); if (comma && comma - uri >= 7 && strncmp(comma - 7, ";base64", 7) == 0) { - cgltf_result res = cgltf_load_buffer_base64(&options, gltf->buffers[i].size, comma + 1, &gltf->buffers[i].data); + cgltf_result res = cgltf_load_buffer_base64(&options, gltf->buffers[i].size, + comma + 1, &gltf->buffers[i].data); if (res != cgltf_result_success) { slog.e << "Unable to load " << uri << io::endl; return false; @@ -469,7 +453,7 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { if (!pImpl->mIgnoreBindTransform) { pImpl->mIgnoreBindTransform = asset->isInstanced(); } - pImpl->cgltfSkinBaseAddress = &gltf->skins[0]; + pImpl->mSkinBaseAddress = &gltf->skins[0]; updateBoundingBoxes(asset); } @@ -533,12 +517,12 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { // we need to generate the contents of a GPU buffer by processing one or more CPU buffer(s). pImpl->computeTangents(asset); - // Non-textured renderables are now considered ready, so notify the dependency graph. - asset->mDependencyGraph.finalize(); - pImpl->mCurrentAsset = asset; - // Finally, create Filament Textures and begin loading image files. - asset->mResourcesLoaded = pImpl->createTextures(async); + asset->mResourcesLoaded = pImpl->createTextures(asset, async); + + // Non-textured renderables are now considered ready, and we can guarantee that no new + // materials or textures will be added. notify the dependency graph. + asset->mDependencyGraph.finalize(); asset->createAnimators(); @@ -546,334 +530,178 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { } bool ResourceLoader::asyncBeginLoad(FilamentAsset* asset) { + pImpl->mAsyncAsset = upcast(asset); return loadResources(upcast(asset), true); } void ResourceLoader::asyncCancelLoad() { pImpl->cancelTextureDecoding(); + pImpl->mAsyncAsset = nullptr; pImpl->mEngine->flushAndWait(); } +void ResourceLoader::addTextureProvider(const char* mimeType, TextureProvider* provider) { + pImpl->mTextureProviders[mimeType] = provider; +} + float ResourceLoader::asyncGetLoadProgress() const { - const float finished = pImpl->mNumDecoderTasksFinished; - const float total = pImpl->mNumDecoderTasks; - return total == 0 ? 0 : finished / total; + if (pImpl->mTextureProviders.empty() || !pImpl->mAsyncAsset) { + return 0; + } + size_t pushedCount = 0; + size_t decodedCount = 0; + for (const auto& iter : pImpl->mTextureProviders) { + pushedCount += iter.second->getPushedCount(); + decodedCount += iter.second->getDecodedCount(); + } + return pushedCount == 0 ? 0 : (float(decodedCount) / pushedCount); } void ResourceLoader::asyncUpdateLoad() { - if (!UTILS_HAS_THREADING) { - pImpl->decodeSingleTexture(); - } - pImpl->uploadPendingTextures(); -} - -void ResourceLoader::Impl::decodeSingleTexture() { - assert(!UTILS_HAS_THREADING); - int w, h, c; - - // Check if any buffer-based textures haven't been decoded yet. - for (auto& pair : mBufferTextureCache) { - const uint8_t* sourceData = (const uint8_t*) pair.first; - TextureCacheEntry* entry = pair.second.get(); - if (entry->texels) { - continue; - } - entry->texels = stbi_load_from_memory(sourceData, entry->bufferSize, &w, &h, &c, 4); + if (!pImpl->mAsyncAsset) { return; } - - // Check if any URI-based textures haven't been decoded yet. - for (auto& pair : mUriTextureCache) { - auto uri = pair.first; - TextureCacheEntry* entry = pair.second.get(); - if (entry->texels) { - continue; + for (const auto& iter : pImpl->mTextureProviders) { + iter.second->updateQueue(); + while (Texture* texture = iter.second->popTexture()) { + pImpl->mAsyncAsset->mDependencyGraph.markAsReady(texture); } - - // First, check the user-supplied resource cache for this URI. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - entry->texels = stbi_load_from_memory(sourceData, iter->second.size, &w, &h, &c, 4); - return; - } - - // Otherwise load it from the file system if this platform supports it. - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - entry->completed = true; - mNumDecoderTasksFinished++; - return; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - entry->texels = stbi_load(fullpath.c_str(), &w, &h, &c, 4); - return; - #endif } } -void ResourceLoader::Impl::uploadPendingTextures() { - auto upload = [this](TextureCacheEntry* entry, Engine& engine) { - Texture* texture = entry->texture; - uint8_t* texels = entry->texels; - if (texture && texels && !entry->completed) { - Texture::PixelBufferDescriptor pbd(texels, - texture->getWidth() * texture->getHeight() * 4, - Texture::Format::RGBA, Texture::Type::UBYTE, FREE_CALLBACK); - texture->setImage(engine, 0, std::move(pbd)); - texture->generateMipmaps(engine); - entry->completed = true; - mNumDecoderTasksFinished++; - mCurrentAsset->mDependencyGraph.markAsReady(texture); - } - }; - for (auto& pair : mBufferTextureCache) upload(pair.second.get(), *mEngine); - for (auto& pair : mUriTextureCache) upload(pair.second.get(), *mEngine); -} - -void ResourceLoader::Impl::releasePendingTextures() { - auto release = [this](TextureCacheEntry* entry, Engine& engine) { - Texture* texture = entry->texture; - uint8_t* texels = entry->texels; - if (texture && texels && !entry->completed) { - // Normally the ownership of these texels is transferred to PixelBufferDescriptor, but - // if uploads have been cancelled then we need to free them explicitly. - free(texels); - } - }; - for (auto& pair : mBufferTextureCache) release(pair.second.get(), *mEngine); - for (auto& pair : mUriTextureCache) release(pair.second.get(), *mEngine); -} - -void ResourceLoader::Impl::addTextureCacheEntry(const TextureSlot& tb) { - TextureCacheEntry* entry = nullptr; - +Texture* ResourceLoader::Impl::getOrCreateTexture(FFilamentAsset* asset, const TextureSlot& tb) { const cgltf_texture* srcTexture = tb.texture; - const cgltf_buffer_view* bv = srcTexture->image->buffer_view; - const char* uri = srcTexture->image->uri; - const uint32_t totalSize = uint32_t(bv ? bv->size : 0); - void** data = bv ? &bv->buffer->data : nullptr; - const size_t offset = bv ? bv->offset : 0; + const cgltf_image* image = srcTexture->image; + const cgltf_buffer_view* bv = image->buffer_view; + const char* uri = image->uri; - // Check if the texture binding uses BufferView data (i.e. it does not have a URI). - if (data) { - const uint8_t* sourceData = offset + (const uint8_t*) *data; - entry = mBufferTextureCache[sourceData] ? mBufferTextureCache[sourceData].get() : nullptr; - if (entry) { - return; - } - entry = (mBufferTextureCache[sourceData] = std::make_unique()).get(); - entry->srgb = tb.srgb; - if (!stbi_info_from_memory(sourceData, totalSize, &entry->width, &entry->height, - &entry->numComponents)) { - slog.e << "Unable to decode BufferView texture: " << stbi_failure_reason() << io::endl; - mBufferTextureCache.erase(sourceData); - return; - } - entry->bufferSize = totalSize; - return; + TextureProvider::FlagBits flags = {}; + if (tb.srgb) { + flags |= int(TextureProvider::Flags::sRGB); } - // Check if we already created a Texture object for this URI. - entry = mUriTextureCache[uri] ? mUriTextureCache[uri].get() : nullptr; - if (entry) { - return; - } - - entry = (mUriTextureCache[uri] = std::make_unique()).get(); - entry->srgb = tb.srgb; - - // Check if this is a data URI. We don't care about the MIME type since stb can infer it. - std::string mimeType; + std::string mime = image->mime_type ? image->mime_type : ""; size_t dataUriSize; - const uint8_t* dataUriContent = parseDataUri(uri, &mimeType, &dataUriSize); - if (dataUriContent) { - BufferDescriptor buffer(dataUriContent, dataUriSize, FREE_CALLBACK); - mUriDataCache.emplace(uri, std::move(buffer)); + const uint8_t* dataUriContent = uri ? parseDataUri(uri, &mime, &dataUriSize) : nullptr; + + if (mime.empty()) { + assert_invariant(uri && "Non-URI images must supply a mime type."); + const std::string extension = Path(uri).getExtension(); + mime = extension == "jpg" ? "image/jpeg" : "image/" + extension; } - // Check the user-supplied resource cache for this URI, otherwise peek at the file. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - if (!stbi_info_from_memory(sourceData, iter->second.size, &entry->width, - &entry->height, &entry->numComponents)) { - slog.e << "Unable to decode " << uri << " : " << stbi_failure_reason() << io::endl; - mUriTextureCache.erase(uri); - } - return; + TextureProvider* provider = mTextureProviders[mime]; + if (!provider) { + slog.e << "Missing texture provider for " << mime << io::endl; + return nullptr; } - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - if (!stbi_info(fullpath.c_str(), &entry->width, &entry->height, &entry->numComponents)) { - slog.e << "Unable to decode " << fullpath.c_str() << " : " << stbi_failure_reason() - << io::endl; - mUriTextureCache.erase(uri); - } - #endif -} -void ResourceLoader::Impl::bindTextureToMaterial(const TextureSlot& tb) { - FFilamentAsset* asset = mCurrentAsset; + Texture* texture = nullptr; - const cgltf_texture* srcTexture = tb.texture; - const cgltf_buffer_view* bv = srcTexture->image->buffer_view; - const char* uri = srcTexture->image->uri; - void** data = bv ? &bv->buffer->data : nullptr; - const size_t offset = bv ? bv->offset : 0; - - // First check if this is a buffer-based texture. - if (data) { - const uint8_t* sourceData = offset + (const uint8_t*) *data; + // Check if the texture slot uses BufferView data. + if (void** bufferViewData = bv ? &bv->buffer->data : nullptr; bufferViewData) { + assert_invariant(!dataUriContent); + const size_t offset = bv ? bv->offset : 0; + const uint8_t* sourceData = offset + (const uint8_t*) *bufferViewData; if (auto iter = mBufferTextureCache.find(sourceData); iter != mBufferTextureCache.end()) { - auto& entry = iter->second; - if (entry.get() && entry->texture) { - asset->bindTexture(tb, entry->texture); - } + return iter->second; + } + const uint32_t totalSize = uint32_t(bv ? bv->size : 0); + if ((texture = provider->pushTexture(sourceData, totalSize, mime.c_str(), flags))) { + mBufferTextureCache[sourceData] = texture; } - return; } - // Next check if this is a URI-based texture. - if (auto iter = mUriTextureCache.find(uri); iter != mUriTextureCache.end()) { - auto& entry = iter->second; - if (entry.get() && entry->texture) { - asset->bindTexture(tb, entry->texture); + // Check if the texture slot is a data URI. + else if (dataUriContent) { + if (auto iter = mBufferTextureCache.find(uri); iter != mBufferTextureCache.end()) { + free((void*)dataUriContent); + return iter->second; + } + if ((texture = provider->pushTexture(dataUriContent, dataUriSize, mime.c_str(), flags))) { + mBufferTextureCache[uri] = texture; + } + free((void*)dataUriContent); + } + + // Check the user-supplied resource cache for this URI. + else if (auto iter = mUriDataCache.find(uri); iter != mUriDataCache.end()) { + const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; + if (auto iter = mBufferTextureCache.find(sourceData); iter != mBufferTextureCache.end()) { + return iter->second; + } + if ((texture = provider->pushTexture(sourceData, iter->second.size, mime.c_str(), flags))) { + mBufferTextureCache[sourceData] = texture; } } + + // Finally, try the file system. + else if constexpr (USE_FILESYSTEM) { + if (auto iter = mFilepathTextureCache.find(uri); iter != mFilepathTextureCache.end()) { + return iter->second; + } + Path fullpath = Path(mGltfPath).getParent() + uri; + if (!fullpath.exists()) { + slog.e << "Unable to open " << fullpath << io::endl; + return nullptr; + } + using namespace std; + ifstream filest(fullpath); + vector buffer; + filest.seekg(0, ios::end); + buffer.reserve((size_t) filest.tellg()); + filest.seekg(0, ios::beg); + buffer.assign((istreambuf_iterator(filest)), istreambuf_iterator()); + if ((texture = provider->pushTexture(buffer.data(), buffer.size(), mime.c_str(), flags))) { + mFilepathTextureCache[uri] = texture; + } + + // If the platform does not have a filesystem, emit an error and move on. + } else { + slog.e << "Unable to load " << uri << io::endl; + return nullptr; + } + + if (!texture) { + slog.e << "Unable to decode texture data: " << provider->getPushMessage() << io::endl; + } + + asset->takeOwnership(texture); + return texture; } void ResourceLoader::Impl::cancelTextureDecoding() { - JobSystem* js = &mEngine->getJobSystem(); - if (mDecoderRootJob) { - js->waitAndRelease(mDecoderRootJob); - mDecoderRootJob = nullptr; + for (const auto& iter : mTextureProviders) { + iter.second->cancelDecoding(); } - releasePendingTextures(); - mBufferTextureCache.clear(); - mUriTextureCache.clear(); - mCurrentAsset = nullptr; - mNumDecoderTasksFinished = 0; - mNumDecoderTasks = 0; + mAsyncAsset = nullptr; } -bool ResourceLoader::Impl::createTextures(bool async) { +bool ResourceLoader::Impl::createTextures(FFilamentAsset* asset, bool async) { // If any decoding jobs are still underway, wait for them to finish. - JobSystem* js = &mEngine->getJobSystem(); - if (mDecoderRootJob) { - js->waitAndRelease(mDecoderRootJob); - mDecoderRootJob = nullptr; + for (const auto& iter : mTextureProviders) { + iter.second->waitForCompletion(); + iter.second->updateQueue(); } - mBufferTextureCache.clear(); - mUriTextureCache.clear(); - - // First, determine texture dimensions and create texture cache entries. - FFilamentAsset* asset = mCurrentAsset; + // Create new texture objects if they are not cached. for (auto slot : asset->mTextureSlots) { - addTextureCacheEntry(slot); - } - - // Tally up the total number of textures that need to be decoded. Zero textures is a special - // case that needs to report 100% progress right away, so we set NumDecoderTasks and Finished - // both to 1. If they were both 0, this would indicate that loading has not started. - mNumDecoderTasks = mBufferTextureCache.size() + mUriTextureCache.size(); - if (mNumDecoderTasks == 0) { - mNumDecoderTasks = 1; - mNumDecoderTasksFinished = 1; - } else { - mNumDecoderTasksFinished = 0; - } - - // Next create blank Filament textures. - auto createTexture = [=](TextureCacheEntry* entry) { - entry->texture = Texture::Builder() - .width(entry->width) - .height(entry->height) - .levels(0xff) - .format(entry->srgb ? Texture::InternalFormat::SRGB8_A8 : Texture::InternalFormat::RGBA8) - .build(*mEngine); - asset->takeOwnership(entry->texture); - }; - for (auto& pair : mBufferTextureCache) createTexture(pair.second.get()); - for (auto& pair : mUriTextureCache) createTexture(pair.second.get()); - - // Bind the textures to material instances. - for (auto slot : asset->mTextureSlots) { - bindTextureToMaterial(slot); - } - - // Before creating jobs for PNG / JPEG decoding, we might need to return early. On single - // threaded systems, it is usually fine to create jobs because the job system will simply - // execute serially. However if the client requests async behavior, then we need to wait - // until subsequent calls to asyncUpdateLoad(). - if (!UTILS_HAS_THREADING && async) { - return true; - } - - JobSystem::Job* parent = js->createJob(); - - // Create a copy of the shared_ptr to the source data to prevent it from being freed during - // the texture decoding process. - FFilamentAsset::SourceHandle retainSourceAsset = asset->mSourceAsset; - - // Kick off jobs that decode texels from buffer pointers. - for (auto& pair : mBufferTextureCache) { - const uint8_t* sourceData = (const uint8_t*) pair.first; - TextureCacheEntry* entry = pair.second.get(); - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, sourceData] { - int width, height, comp; - entry->texels = stbi_load_from_memory(sourceData, entry->bufferSize, - &width, &height, &comp, 4); - }); - js->run(decode); - } - - // Kick off jobs that decode texels from URI strings. - for (auto& pair : mUriTextureCache) { - auto uri = pair.first; - TextureCacheEntry* entry = pair.second.get(); - - // First, check the user-supplied resource cache for this URI. - auto iter = mUriDataCache.find(uri); - if (iter != mUriDataCache.end()) { - const uint8_t* sourceData = (const uint8_t*) iter->second.buffer; - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, sourceData, iter] { - int width, height, comp; - entry->texels = stbi_load_from_memory(sourceData, iter->second.size, &width, - &height, &comp, 4); - }); - js->run(decode); - continue; + if (Texture* texture = getOrCreateTexture(asset, slot)) { + asset->bindTexture(slot, texture); } - - // Otherwise load it from the file system if this platform supports it. - #if !USE_FILESYSTEM - slog.e << "Unable to load texture: " << uri << io::endl; - return false; - #else - Path fullpath = Path(mGltfPath).getParent() + uri; - JobSystem::Job* decode = jobs::createJob(*js, parent, [retainSourceAsset, entry, fullpath] { - int width, height, comp; - entry->texels = stbi_load(fullpath.c_str(), &width, &height, &comp, 4); - }); - js->run(decode); - #endif } + // Non-threaded systems are required to use the asynchronous API. + assert_invariant(UTILS_HAS_THREADING || async); + if (async) { - mDecoderRootJob = js->runAndRetain(parent); return true; } - // Wait for decoding to finish. - js->runAndWait(parent); - - // Finally, upload texels to the GPU and generate mipmaps. - mCurrentAsset = asset; - uploadPendingTextures(); + for (const auto& iter : mTextureProviders) { + iter.second->waitForCompletion(); + iter.second->updateQueue(); + } return true; } @@ -971,8 +799,8 @@ void ResourceLoader::Impl::computeTangents(FFilamentAsset* asset) { } ResourceLoader::Impl::~Impl() { - if (mDecoderRootJob) { - mEngine->getJobSystem().waitAndRelease(mDecoderRootJob); + for (const auto& iter : mTextureProviders) { + iter.second->cancelDecoding(); } } @@ -1078,15 +906,18 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { for (cgltf_size slot = 0; slot < prim->attributes_count; slot++) { const cgltf_attribute& attr = prim->attributes[slot]; const cgltf_accessor* accessor = attr.data; - if (attr.type == cgltf_attribute_type_position && cgltf_num_components(accessor->type) >= posAttrSize) { + if (attr.type == cgltf_attribute_type_position && + cgltf_num_components(accessor->type) >= posAttrSize) { verts.resize(accessor->count * posAttrSize); cgltf_accessor_unpack_floats(accessor, &verts[0], accessor->count * posAttrSize); } - if (attr.type == cgltf_attribute_type_joints && cgltf_num_components(accessor->type) >= skinningAttrSize) { + if (attr.type == cgltf_attribute_type_joints && + cgltf_num_components(accessor->type) >= skinningAttrSize) { rawJoints.resize(accessor->count * skinningAttrSize); cgltf_accessor_unpack_floats(accessor, &rawJoints[0], accessor->count * skinningAttrSize); } - if (attr.type == cgltf_attribute_type_weights && cgltf_num_components(accessor->type) >= skinningAttrSize) { + if (attr.type == cgltf_attribute_type_weights && + cgltf_num_components(accessor->type) >= skinningAttrSize) { weights.resize(accessor->count * skinningAttrSize); cgltf_accessor_unpack_floats(accessor, &weights[0], accessor->count * skinningAttrSize); } @@ -1109,7 +940,8 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { if (!pImpl->mNormalizeSkinningWeights) { skinMatrix /= skinMatrix[3].w; } - float3 skinnedPoint = (point.x * skinMatrix[0] + point.y * skinMatrix[1] + point.z * skinMatrix[2] + skinMatrix[3]).xyz; + float3 skinnedPoint = (point.x * skinMatrix[0] + + point.y * skinMatrix[1] + point.z * skinMatrix[2] + skinMatrix[3]).xyz; aabb.min = min(aabb.min, skinnedPoint); aabb.max = max(aabb.max, skinnedPoint); } @@ -1117,7 +949,7 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { *result = aabb; }; - // Collect all mesh primitives that we wish to find bounds for. For each mesh primitive, we also + // Collect all mesh primitives that we wish to find bounds for. For each mesh primitive, we also // collect the skin it bound to (nullptr if not skinned) for bounds computation. std::vector> primitives; for (auto iter : nodeMap) { @@ -1126,7 +958,7 @@ void ResourceLoader::updateBoundingBoxes(FFilamentAsset* asset) const { if (cgltfSkin) { // importSkins unpacked cgltfSkin into FFilamentInstance::SkinVector bijectively so that // the unpacked Skin can be retrieved given cgltfSkin index - int skinIndex = cgltfSkin - pImpl->cgltfSkinBaseAddress; + int skinIndex = cgltfSkin - pImpl->mSkinBaseAddress; skin = &asset->mSkins[skinIndex]; } const cgltf_mesh* cgltfMesh = iter.first->mesh; diff --git a/libs/gltfio/src/StbProvider.cpp b/libs/gltfio/src/StbProvider.cpp new file mode 100644 index 0000000000..8b1efce2ac --- /dev/null +++ b/libs/gltfio/src/StbProvider.cpp @@ -0,0 +1,261 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include + +#include +#include + +#include + +using namespace filament; +using namespace utils; + +using std::atomic; +using std::vector; +using std::unique_ptr; + +namespace gltfio { + +class StbProvider : public TextureProvider { +public: + StbProvider(Engine* engine); + ~StbProvider(); + + Texture* pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, uint64_t flags) final; + + Texture* popTexture() final; + void updateQueue() final; + void waitForCompletion() final; + void cancelDecoding() final; + const char* getPushMessage() const final; + const char* getPopMessage() const final; + size_t getPushedCount() const final { return mPushedCount; } + size_t getPoppedCount() const final { return mPoppedCount; } + size_t getDecodedCount() const final { return mDecodedCount; } + +private: + enum class TextureState { + DECODING, // Texture has been pushed, mipmap levels are not yet complete. + READY, // Mipmap levels are available but texture has not been popped yet. + POPPED, // Client has popped the texture from the queue. + }; + + struct TextureInfo { + Texture* texture; + TextureState state; + atomic decodedTexelsBaseMipmap; + vector sourceBuffer; + JobSystem::Job* decoderJob; + }; + + // Declare some sentinel values for the "decodedTexelsBaseMipmap" field. + // Note that the "state" field can be modified only on the foreground thread. + static const intptr_t DECODING_NOT_READY = 0x0; + static const intptr_t DECODING_ERROR = 0x1; + + void decodeSingleTexture(); + + size_t mPushedCount = 0; + size_t mPoppedCount = 0; + size_t mDecodedCount = 0; + vector > mTextures; + JobSystem::Job* mDecoderRootJob; + std::string mRecentPushMessage; + std::string mRecentPopMessage; + Engine* const mEngine; +}; + +Texture* StbProvider::pushTexture(const uint8_t* data, size_t byteCount, + const char* mimeType, FlagBits flags) { + int width, height, numComponents; + if (!stbi_info_from_memory(data, byteCount, &width, &height, &numComponents)) { + mRecentPushMessage = std::string("Unable to parse texture: ") + stbi_failure_reason(); + return nullptr; + } + + using InternalFormat = Texture::InternalFormat; + const FlagBits sRGB = FlagBits(Flags::sRGB); + + Texture* texture = Texture::Builder() + .width(width) + .height(height) + .levels(0xff) + .format((flags & sRGB) ? InternalFormat::SRGB8_A8 : InternalFormat::RGBA8) + .build(*mEngine); + + if (texture == nullptr) { + mRecentPushMessage = "Unable to build Texture object."; + return nullptr; + } + + mRecentPushMessage.clear(); + TextureInfo* info = mTextures.emplace_back(new TextureInfo).get(); + ++mPushedCount; + + info->texture = texture; + info->state = TextureState::DECODING; + info->sourceBuffer.assign(data, data + byteCount); + info->decodedTexelsBaseMipmap.store(DECODING_NOT_READY); + + // On single threaded systems, it is usually fine to create jobs because the job system will + // simply execute serially. However in our case, we wish to amortize the decoder cost across + // several frames, so we instead use the updateQueue() method to perform decoding. + if constexpr (!UTILS_HAS_THREADING) { + info->decoderJob = nullptr; + return texture; + } + + JobSystem* js = &mEngine->getJobSystem(); + info->decoderJob = jobs::createJob(*js, mDecoderRootJob, [this, info] { + auto& source = info->sourceBuffer; + int width, height, comp; + + // Test asynchronous loading by uncommenting this line. + // std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 10000)); + + stbi_uc* texels = stbi_load_from_memory(source.data(), source.size(), + &width, &height, &comp, 4); + source.clear(); + source.shrink_to_fit(); + info->decodedTexelsBaseMipmap.store(texels ? intptr_t(texels) : DECODING_ERROR); + }); + + js->runAndRetain(info->decoderJob); + return texture; +} + +Texture* StbProvider::popTexture() { + // We don't bother shrinking the mTextures vector here, instead we periodically clean it up in + // the updateQueue method, since popTexture is typically called more frequently. Textures + // can become ready in non-deterministic order due to concurrency. + for (auto& texture : mTextures) { + if (texture->state == TextureState::READY) { + texture->state = TextureState::POPPED; + ++mPoppedCount; + const intptr_t ptr = texture->decodedTexelsBaseMipmap.load(); + if (ptr == DECODING_ERROR || ptr == DECODING_NOT_READY) { + mRecentPopMessage = "Texture is incomplete"; + } else { + mRecentPopMessage.clear(); + } + return texture->texture; + } + } + return nullptr; +} + +void StbProvider::updateQueue() { + if (!UTILS_HAS_THREADING) { + decodeSingleTexture(); + } + JobSystem* js = &mEngine->getJobSystem(); + for (auto& info : mTextures) { + if (info->state != TextureState::DECODING) { + continue; + } + Texture* texture = info->texture; + if (intptr_t data = info->decodedTexelsBaseMipmap.load()) { + if (info->decoderJob) { + js->waitAndRelease(info->decoderJob); + } + if (data == DECODING_ERROR) { + info->state = TextureState::READY; + ++mDecodedCount; + continue; + } + Texture::PixelBufferDescriptor pbd((uint8_t*) data, + texture->getWidth() * texture->getHeight() * 4, Texture::Format::RGBA, + Texture::Type::UBYTE, [](void* mem, size_t, void*) { stbi_image_free(mem); }); + texture->setImage(*mEngine, 0, std::move(pbd)); + + // Call generateMipmaps unconditionally to fulfill the promise of the TextureProvider + // interface. Providers of hierarchical images (e.g. KTX) call this only if needed. + texture->generateMipmaps(*mEngine); + + info->state = TextureState::READY; + ++mDecodedCount; + } + } + + // Here we periodically clean up the "queue" (which is really just a vector) by removing unused + // items from the front. This might ignore a popped texture that occurs in the middle of the + // vector, but that's okay, it will be cleaned up eventually. + decltype(mTextures)::iterator last = mTextures.begin(); + while (last != mTextures.end() && (*last)->state == TextureState::POPPED) ++last; + mTextures.erase(mTextures.begin(), last); +} + +void StbProvider::waitForCompletion() { + JobSystem& js = mEngine->getJobSystem(); + for (auto& info : mTextures) { + if (info->decoderJob) { + js.waitAndRelease(info->decoderJob); + } + } +} + +void StbProvider::cancelDecoding() { + // TODO: Currently, StbProvider runs jobs eagerly and JobSystem does not allow cancellation of + // in-flight jobs. We should consider throttling the number of simultaneous decoder jobs, which + // would allow for actual cancellation. + waitForCompletion(); +} + +const char* StbProvider::getPushMessage() const { + return mRecentPushMessage.empty() ? nullptr : mRecentPushMessage.c_str(); +} + +const char* StbProvider::getPopMessage() const { + return mRecentPopMessage.empty() ? nullptr : mRecentPopMessage.c_str(); +} + +void StbProvider::decodeSingleTexture() { + assert_invariant(!UTILS_HAS_THREADING); + for (auto& info : mTextures) { + if (info->state == TextureState::DECODING) { + auto& source = info->sourceBuffer; + int width, height, comp; + stbi_uc* texels = stbi_load_from_memory(source.data(), source.size(), + &width, &height, &comp, 4); + source.clear(); + source.shrink_to_fit(); + info->decodedTexelsBaseMipmap.store(texels ? intptr_t(texels) : DECODING_ERROR); + break; + } + } +} + +StbProvider::StbProvider(Engine* engine) : mEngine(engine) { + mDecoderRootJob = mEngine->getJobSystem().createJob(); +} + +StbProvider::~StbProvider() { + cancelDecoding(); + mEngine->getJobSystem().release(mDecoderRootJob); +} + +TextureProvider* createStbProvider(Engine* engine) { + return new StbProvider(engine); +} + +} // namespace gltfio diff --git a/libs/viewer/include/viewer/AutomationEngine.h b/libs/viewer/include/viewer/AutomationEngine.h index ac8b98f721..5617d82b6f 100644 --- a/libs/viewer/include/viewer/AutomationEngine.h +++ b/libs/viewer/include/viewer/AutomationEngine.h @@ -40,7 +40,7 @@ namespace viewer { * * When executing a test, clients should call tick() after each frame is rendered, which gives * automation an opportunity to push settings to Filament, increment the current test index (if - * enough time has elapsed), and request an asychronous screenshot. + * enough time has elapsed), and request an asynchronous screenshot. * * The time to sleep between tests is configurable and can be set to zero. Automation also waits a * specified minimum number of frames between tests. diff --git a/samples/gltf_viewer.cpp b/samples/gltf_viewer.cpp index a2cae36747..92cb656651 100644 --- a/samples/gltf_viewer.cpp +++ b/samples/gltf_viewer.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,7 @@ struct App { MaterialSource materialSource = GENERATE_SHADERS; gltfio::ResourceLoader* resourceLoader = nullptr; + gltfio::TextureProvider* decoder = nullptr; bool recomputeAabb = false; bool ignoreBindTransform = false; @@ -423,6 +425,9 @@ int main(int argc, char** argv) { configuration.normalizeSkinningWeights = true; if (!app.resourceLoader) { app.resourceLoader = new gltfio::ResourceLoader(configuration); + app.decoder = createStbProvider(app.engine); + app.resourceLoader->addTextureProvider("image/png", app.decoder); + app.resourceLoader->addTextureProvider("image/jpeg", app.decoder); } app.resourceLoader->asyncBeginLoad(app.asset); app.asset->releaseSourceData(); @@ -644,6 +649,7 @@ int main(int argc, char** argv) { delete app.viewer; delete app.materials; delete app.names; + delete app.decoder; AssetLoader::destroy(&app.assetLoader); }; diff --git a/third_party/stb/stb_image.cpp b/third_party/stb/stb_image.cpp new file mode 100644 index 0000000000..3ee30bdfb5 --- /dev/null +++ b/third_party/stb/stb_image.cpp @@ -0,0 +1,9 @@ +#define STBI_NO_BMP +#define STBI_NO_PSD +#define STBI_NO_TGA +#define STBI_NO_GIF +#define STBI_NO_PIC +#define STBI_NO_PNM + +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" diff --git a/third_party/stb/tnt/CMakeLists.txt b/third_party/stb/tnt/CMakeLists.txt index b952728584..330ff4f694 100644 --- a/third_party/stb/tnt/CMakeLists.txt +++ b/third_party/stb/tnt/CMakeLists.txt @@ -2,12 +2,15 @@ cmake_minimum_required(VERSION 3.19) project(stb) set(TARGET stb) -set(OUR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -set(PUBLIC_HDR_DIR ${OUR_DIR}) -add_library(${TARGET} INTERFACE) +set(PUBLIC_HDR_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_include_directories(${TARGET} INTERFACE ${PUBLIC_HDR_DIR}) +set(PUBLIC_HDRS ../stb_image.h) -file(GLOB_RECURSE PUBLIC_HDRS ${PUBLIC_HDR_DIR}/*.h) -target_sources(${TARGET} INTERFACE ${PUBLIC_HDRS}) +set(SRCS ../stb_image.cpp) + +add_library(${TARGET} STATIC ${PUBLIC_HDRS} ${SRCS}) + +target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) + +install(TARGETS stb ARCHIVE DESTINATION lib/${DIST_DIR}) diff --git a/web/filament-js/extensions.js b/web/filament-js/extensions.js index 519095e4fa..7e0c58f0ce 100644 --- a/web/filament-js/extensions.js +++ b/web/filament-js/extensions.js @@ -643,6 +643,12 @@ Filament.loadClassExtensions = function() { config.normalizeSkinningWeights, config.recomputeBoundingBoxes, config.ignoreBindTransform); + + const stbProvider = new Filament.gltfio$StbProvider(engine); + + resourceLoader.addTextureProvider("image/jpeg", stbProvider); + resourceLoader.addTextureProvider("image/png", stbProvider); + const onComplete = () => { resourceLoader.asyncBeginLoad(asset); @@ -657,6 +663,7 @@ Filament.loadClassExtensions = function() { if (progress >= 1) { clearInterval(timer); resourceLoader.delete(); + stbProvider.delete(); onDone(); } }, interval); diff --git a/web/filament-js/jsbindings.cpp b/web/filament-js/jsbindings.cpp index c81b23da4f..1f1ce30ca2 100644 --- a/web/filament-js/jsbindings.cpp +++ b/web/filament-js/jsbindings.cpp @@ -64,9 +64,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -80,6 +80,8 @@ #include #include +#include + #include #include @@ -1886,18 +1888,28 @@ class_("gltfio$FilamentInstance") .function("applyMaterialVariant", &FilamentInstance::applyMaterialVariant) .function("getAnimator", &FilamentInstance::getAnimator, allow_raw_pointers()); -// This little wrapper exists to get around RTTI requirements in embind. +// These little wrappers exist to get around RTTI requirements in embind. + struct UbershaderLoader { MaterialProvider* provider; void destroyMaterials() { provider->destroyMaterials(); } }; +struct StbProvider { + TextureProvider* provider; +}; + class_("gltfio$UbershaderLoader") .constructor(EMBIND_LAMBDA(UbershaderLoader, (Engine* engine), { return UbershaderLoader { createUbershaderLoader(engine) }; })) .function("destroyMaterials", &UbershaderLoader::destroyMaterials); +class_("gltfio$StbProvider") + .constructor(EMBIND_LAMBDA(StbProvider, (Engine* engine), { + return StbProvider { createStbProvider(engine) }; + })); + class_("gltfio$AssetLoader") .constructor(EMBIND_LAMBDA(AssetLoader*, (Engine* engine, UbershaderLoader materials), { @@ -1959,6 +1971,11 @@ class_("gltfio$ResourceLoader") self->addResourceData(url.c_str(), std::move(*buffer.bd)); }), allow_raw_pointers()) + .function("addTextureProvider", EMBIND_LAMBDA(void, (ResourceLoader* self, std::string mime, + StbProvider provider), { + self->addTextureProvider(mime.c_str(), provider.provider); + }), allow_raw_pointers()) + .function("hasResourceData", EMBIND_LAMBDA(bool, (ResourceLoader* self, std::string url), { return self->hasResourceData(url.c_str()); }), allow_raw_pointers())