diff --git a/.github/actions/verify-release-notes/Dockerfile b/.github/actions/verify-release-notes/Dockerfile new file mode 100644 index 0000000000..2bdf2021e9 --- /dev/null +++ b/.github/actions/verify-release-notes/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10.4 + +RUN pip3 install pygithub==1.55 + +ENV PYTHONUNBUFFERED=1 + +COPY verify_release_notes.py /verify_release_notes.py + +RUN chmod +x /verify_release_notes.py +ENTRYPOINT [ "/verify_release_notes.py" ] diff --git a/.github/actions/verify-release-notes/action.yml b/.github/actions/verify-release-notes/action.yml new file mode 100644 index 0000000000..4838b5ae80 --- /dev/null +++ b/.github/actions/verify-release-notes/action.yml @@ -0,0 +1,35 @@ +name: 'Verify Release Notes' +description: 'Verifies that a RELEASE_NOTES file has been modified' +inputs: + repo-token: + description: 'The GitHub token' + required: true + pull-request-number: + description: 'The Pull Request number' + required: true + bypass-label-name: + description: 'The Label used to bypass this check' + required: false + default: 'internal' + release-notes-file: + description: 'The path to the RELEASE_NOTES file' + required: false + default: 'RELEASE_NOTES.md' + pull-request-repo-full-name: + description: 'The full name of the Pull Request repo' + required: false + default: ${{ github.event.pull_request.head.repo.full_name }} + pull-request-head-ref: + description: 'The HEAD ref of the Pull Request' + required: false + default: ${{ github.event.pull_request.head.ref }} +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.repo-token }} + - ${{ inputs.pull-request-number }} + - ${{ inputs.bypass-label-name }} + - ${{ inputs.release-notes-file }} + - ${{ inputs.pull-request-repo-full-name }} + - ${{ inputs.pull-request-head-ref }} diff --git a/.github/actions/verify-release-notes/verify_release_notes.py b/.github/actions/verify-release-notes/verify_release_notes.py new file mode 100755 index 0000000000..08defb35a7 --- /dev/null +++ b/.github/actions/verify-release-notes/verify_release_notes.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from github import Github +import sys, os + +def print_usage(): + print('Verify that a GitHub pull request modifies a RELEASE_NOTES file') + print() + print('Usage:') + print(' verify_release_notes.py [arguments]') + print() + print('Arguments:') + print('1. github token') + print('2. pull request number') + print('3. bypass label name') + print('4. release notes file path') + print('5. pull request repo full name') + print('6. pull request head ref') + print() + print('The GITHUB_REPOSITORY environment variable must be set (e.g., google/filament).') + +def leave_single_comment(pull_request, comment_body): + """ Leaves a comment on a PR once, without leaving a duplicate comment. """ + # To avoid spamming the PR author, we'll use this comment tag (which will render invisibly on + # GitHub) to check if we've already left a comment on this PR. + COMMENT_TAG = '\n' + comments = pull_request.get_issue_comments() + for comment in comments: + if comment.body.find(COMMENT_TAG) != -1: + return + # The GitHub token may not have WRITE permissions to leave a comment (for example, for 3P + # contributors). In that case, we simply won't leave a comment. + try: + pull_request.create_issue_comment(f'{COMMENT_TAG}{comment_body}') + except: + print("Unable to leave comment. Continuing.") + +# The first argument is the path to this script. +if len(sys.argv) != 7: + print_usage() + sys.exit(1) + +authentication_token = sys.argv[1] +pull_number = sys.argv[2] +bypass_label_name = sys.argv[3] +release_notes_file = sys.argv[4] +pr_repo_full_name = sys.argv[5] +pr_head_ref = sys.argv[6] + +g = Github(authentication_token) + +repo_name = os.environ.get('GITHUB_REPOSITORY') +if repo_name is None: + print("The GITHUB_REPOSITORY environment variable must be set.") + sys.exit(1) + +repo = g.get_repo(repo_name) + +pull_request = repo.get_pull(int(pull_number)) + +# First check if the PR has the "bypass" label. This label is used for PRs that don't need to update +# RELEASE_NOTES. If so, we can exit immediately. +labels = [l.name for l in pull_request.labels] +if bypass_label_name in labels: + print(f"PR number {pull_number} in repo {repo_name} contains the '{bypass_label_name}' label.") + print("Exiting with success.") + sys.exit(0) + +# Next, check if the release notes file (RELEASE_NOTES.md or similar) has been modified. +files = pull_request.get_files() +for file in files: + if file.filename == release_notes_file: + print(f"PR number {pull_number} in repo {repo_name} modifies '{release_notes_file}'.") + print("Exiting with success.") + sys.exit(0) + +# At this point, we issue a warning to the PR author to remember to modify the release notes, and +# exit with failure. +edit_url = f"https://github.com/{pr_repo_full_name}/edit/{pr_head_ref}/{release_notes_file}" +comment = (f"Please add a release note line to [{release_notes_file}]({edit_url}). " + f"If this PR does not warrant a release note, add the '{bypass_label_name}' label " + f"to this PR.") +print(comment) +leave_single_comment(pull_request, comment) + +sys.exit(1) diff --git a/.github/workflows/android-continuous.yml b/.github/workflows/android-continuous.yml index 5c6db61641..ecf4548818 100644 --- a/.github/workflows/android-continuous.yml +++ b/.github/workflows/android-continuous.yml @@ -33,10 +33,6 @@ jobs: with: name: gltfio-android-release path: out/gltfio-android-release.aar - - uses: actions/upload-artifact@v1.0.0 - with: - name: gltfio-android-lite-release - path: out/gltfio-android-lite-release.aar - uses: actions/upload-artifact@v1.0.0 with: name: filament-utils-android-release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87a54aa136..3e0097555b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -126,7 +126,6 @@ jobs: mv out/filamat-android-release.aar out/filamat-${TAG}-android.aar mv out/filamat-android-lite-release.aar out/filamat-${TAG}-lite-android.aar mv out/gltfio-android-release.aar out/gltfio-${TAG}-android.aar - mv out/gltfio-android-lite-release.aar out/gltfio-${TAG}-lite-android.aar mv out/filament-utils-android-release.aar out/filament-utils-${TAG}-android.aar python3 build/common/upload-assets.py ${TAG} out/*.aar out/*.apk env: diff --git a/.github/workflows/verify-release-notes.yml b/.github/workflows/verify-release-notes.yml new file mode 100644 index 0000000000..f490136448 --- /dev/null +++ b/.github/workflows/verify-release-notes.yml @@ -0,0 +1,26 @@ +name: Verify Release Notes + +on: + pull_request: + branches: + - main + types: [opened, synchronize, reopened, labeled] + +jobs: + verify-release-notes: + name: Verify Release Notes + runs-on: ubuntu-latest + steps: + # We *only* need to check out the .github/ directory. + # The verify RELEASE_NOTES script uses the GitHub API to verify that RELEASE_NOTES was + # modified. + - name: Check out action + uses: Bhacaz/checkout-files@73e17cfbe8d7e0c6b2672b20cb05a718e20d18d4 + with: + files: .github + token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify release notes + uses: ./.github/actions/verify-release-notes + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + pull-request-number: ${{ github.event.pull_request.number }} diff --git a/CMakeLists.txt b/CMakeLists.txt index a074b136e5..ab1a1d411a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -650,6 +650,7 @@ add_subdirectory(${LIBRARIES}/image) add_subdirectory(${LIBRARIES}/ktxreader) add_subdirectory(${LIBRARIES}/math) add_subdirectory(${LIBRARIES}/mathio) +add_subdirectory(${LIBRARIES}/uberz) add_subdirectory(${LIBRARIES}/utils) add_subdirectory(${LIBRARIES}/viewer) add_subdirectory(${FILAMENT}/filament) @@ -721,9 +722,10 @@ if (IS_HOST_PLATFORM) add_subdirectory(${TOOLS}/rgb-to-lmsr) add_subdirectory(${TOOLS}/roughness-prefilter) add_subdirectory(${TOOLS}/specular-color) + add_subdirectory(${TOOLS}/uberz) endif() # Generate exported executables for cross-compiled builds (Android, WebGL, and iOS) if (NOT CMAKE_CROSSCOMPILING) - export(TARGETS matc cmgen filamesh mipgen resgen glslminifier FILE ${IMPORT_EXECUTABLES}) + export(TARGETS matc cmgen filamesh mipgen resgen uberz glslminifier FILE ${IMPORT_EXECUTABLES}) endif() diff --git a/README.md b/README.md index b4dba5bc70..2ed3361bbf 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ repositories { } dependencies { - implementation 'com.google.android.filament:filament-android:1.23.1' + implementation 'com.google.android.filament:filament-android:1.23.2' } ``` @@ -51,7 +51,7 @@ Here are all the libraries available in the group `com.google.android.filament`: iOS projects can use CocoaPods to install the latest release: ``` -pod 'Filament', '~> 1.23.1' +pod 'Filament', '~> 1.23.2' ``` ### Snapshots diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fb0eb585a7..8ac8b8ec81 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,6 +5,18 @@ A new header is inserted each time a *tag* is created. ## main branch +## v1.23.2 + +- gltfio: Fix morphing for un-packed accessors. +- gltfio: Ubershaders are now packaged into flexible archives. +- gltfio: Remove poorly maintained lite flavor. +- engine: Disable user scissor while rendering the Shadow Maps. +- engine: Merge identical backend `RenderPrimitives` together. +- engine: Improve `ResourceAllocator` performance a bit by reserving 128 cache entries. +- utils: Remove `std::hash` definitions for `libutils` types. Use `T::Hasher` explicitly instead. [⚠️ **API Change**] +- backend: Fix WGL context attributes. +- Metal: Fix potential invalid shaders when using gltfio in Ubershader mode. [⚠️ **Recompile Materials to get the fix**] + ## v1.23.1 - gltfio: support skinning with bones that do not belong to any scene. diff --git a/android/gltfio-android/CMakeLists.txt b/android/gltfio-android/CMakeLists.txt index 5ad0e42b43..c55b1ff358 100644 --- a/android/gltfio-android/CMakeLists.txt +++ b/android/gltfio-android/CMakeLists.txt @@ -31,14 +31,14 @@ add_library(utils STATIC IMPORTED) set_target_properties(utils PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libutils.a) +add_library(uberzlib STATIC IMPORTED) +set_target_properties(uberzlib PROPERTIES IMPORTED_LOCATION + ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libuberzlib.a) + add_library(gltfio_resources STATIC IMPORTED) set_target_properties(gltfio_resources PROPERTIES IMPORTED_LOCATION ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libgltfio_resources.a) -add_library(gltfio_resources_lite STATIC IMPORTED) -set_target_properties(gltfio_resources_lite PROPERTIES IMPORTED_LOCATION - ${FILAMENT_DIR}/lib/${ANDROID_ABI}/libgltfio_resources_lite.a) - set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libgltfio-jni.map") set(GLTFIO_SRCS @@ -85,7 +85,6 @@ set(GLTFIO_SRCS src/main/cpp/UbershaderLoader.cpp src/main/cpp/ResourceLoader.cpp - ${FILAMENT_DIR}/include/gltfio/resources/gltfresources_lite.h ${FILAMENT_DIR}/include/gltfio/resources/gltfresources.h ../common/NioUtils.cpp @@ -109,16 +108,8 @@ add_library(gltfio-jni SHARED ${GLTFIO_SRCS}) target_include_directories(gltfio-jni PRIVATE ${GLTFIO_INCLUDE_DIRS}) set_target_properties(gltfio-jni PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/libgltfio-jni.symbols) set_target_properties(gltfio-jni PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/libgltfio-jni.map) - -if(GLTFIO_LITE) - target_compile_definitions(gltfio-jni PUBLIC GLTFIO_LITE=1) - target_link_libraries(gltfio-jni filament-jni utils log stb ktxreader basis_transcoder zstd gltfio_resources_lite) -else() - target_link_libraries(gltfio-jni filament-jni utils log stb ktxreader basis_transcoder zstd gltfio_resources) - - # Enable Draco in the non-lite variant of gltfio. - target_link_libraries(gltfio-jni dracodec) - target_compile_definitions(gltfio-jni PUBLIC GLTFIO_DRACO_SUPPORTED=1) - target_include_directories(gltfio-jni PRIVATE ${DRACO_DIR}/src) - target_include_directories(gltfio-jni PRIVATE ${DRACO_DIR}/tnt) -endif() +target_link_libraries(gltfio-jni filament-jni utils uberzlib log stb ktxreader basis_transcoder zstd gltfio_resources) +target_link_libraries(gltfio-jni dracodec) +target_compile_definitions(gltfio-jni PUBLIC GLTFIO_DRACO_SUPPORTED=1) +target_include_directories(gltfio-jni PRIVATE ${DRACO_DIR}/src) +target_include_directories(gltfio-jni PRIVATE ${DRACO_DIR}/tnt) diff --git a/android/gltfio-android/build.gradle b/android/gltfio-android/build.gradle index 0131c4b51c..a4a3113fc7 100644 --- a/android/gltfio-android/build.gradle +++ b/android/gltfio-android/build.gradle @@ -6,16 +6,6 @@ android { full { dimension "functionality" } - - lite { - dimension "functionality" - - externalNativeBuild { - cmake { - arguments.add("-DGLTFIO_LITE=ON") - } - } - } } packagingOptions { // No need to package up the following shared libs, which arise as a side effect of our @@ -43,11 +33,6 @@ afterEvaluate { project -> artifactId = POM_ARTIFACT_ID_FULL from components.fullRelease } - - liteRelease(MavenPublication) { - artifactId = POM_ARTIFACT_ID_LITE - from components.liteRelease - } } } } diff --git a/android/gradle.properties b/android/gradle.properties index 3cf81f4ca7..4d26074d5e 100644 --- a/android/gradle.properties +++ b/android/gradle.properties @@ -1,5 +1,5 @@ GROUP=com.google.android.filament -VERSION_NAME=1.23.1 +VERSION_NAME=1.23.2 POM_DESCRIPTION=Real-time physically based rendering engine for Android. diff --git a/build.sh b/build.sh index 271f116cf4..00a4faf2ed 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ set -e # Host tools required by Android, WebGL, and iOS builds -MOBILE_HOST_TOOLS="matc resgen cmgen filamesh" +MOBILE_HOST_TOOLS="matc resgen cmgen filamesh uberz" WEB_HOST_TOOLS="${MOBILE_HOST_TOOLS} mipgen filamesh" function print_help { @@ -478,14 +478,12 @@ function build_android { if [[ "${INSTALL_COMMAND}" ]]; then echo "Installing out/filamat-android-debug.aar..." - cp filamat-android/build/outputs/aar/filamat-android-lite-debug.aar ../out/ cp filamat-android/build/outputs/aar/filamat-android-full-debug.aar ../out/filamat-android-debug.aar echo "Installing out/filament-android-debug.aar..." cp filament-android/build/outputs/aar/filament-android-debug.aar ../out/ echo "Installing out/gltfio-android-debug.aar..." - cp gltfio-android/build/outputs/aar/gltfio-android-lite-debug.aar ../out/ cp gltfio-android/build/outputs/aar/gltfio-android-full-debug.aar ../out/gltfio-android-debug.aar echo "Installing out/filament-utils-android-debug.aar..." diff --git a/build/windows/build-github.bat b/build/windows/build-github.bat index ce02c0f7ba..53f719c936 100644 --- a/build/windows/build-github.bat +++ b/build/windows/build-github.bat @@ -120,7 +120,6 @@ cmake ..\.. ^ @echo on cmake --build . --target filagui --config %config% || exit /b cmake --build . --target gltfio_resources --config %config% || exit /b -cmake --build . --target gltfio_resources_lite --config %config% || exit /b cmake --build . --target gltf-resources --config %config% || exit /b cmake --build . --target filamentapp-resources --config %config% || exit /b cmake --build . --target sample-resources --config %config% || exit /b diff --git a/filament/CMakeLists.txt b/filament/CMakeLists.txt index c4cb2a3f84..c001d68053 100644 --- a/filament/CMakeLists.txt +++ b/filament/CMakeLists.txt @@ -63,6 +63,7 @@ set(SRCS src/FrameSkipper.cpp src/Froxelizer.cpp src/Frustum.cpp + src/HwRenderPrimitiveFactory.cpp src/IndexBuffer.cpp src/IndirectLight.cpp src/LightManager.cpp @@ -140,6 +141,7 @@ set(PRIVATE_HDRS src/FrameInfo.h src/FrameSkipper.h src/Froxelizer.h + src/HwRenderPrimitiveFactory.h src/Intersections.h src/MaterialParser.h src/PerViewUniforms.h diff --git a/filament/backend/include/private/backend/Driver.h b/filament/backend/include/private/backend/Driver.h index 58f543ee8a..ab62179c60 100644 --- a/filament/backend/include/private/backend/Driver.h +++ b/filament/backend/include/private/backend/Driver.h @@ -26,6 +26,8 @@ #include +#include + #include // Command debugging off. debugging virtuals are not called. diff --git a/filament/backend/include/private/backend/SamplerGroup.h b/filament/backend/include/private/backend/SamplerGroup.h index f48952235f..7e4ebbd2af 100644 --- a/filament/backend/include/private/backend/SamplerGroup.h +++ b/filament/backend/include/private/backend/SamplerGroup.h @@ -17,17 +17,18 @@ #ifndef TNT_FILAMENT_BACKEND_PRIVATE_SAMPLERGROUP_H #define TNT_FILAMENT_BACKEND_PRIVATE_SAMPLERGROUP_H -#include -#include - #include #include #include #include -namespace filament { -namespace backend { +#include +#include + +#include + +namespace filament::backend { class SamplerGroup { public: @@ -151,7 +152,6 @@ private: mutable utils::bitset32 mDirty; }; -} // namespace backend -} // namespace filament +} // namespace filament::backend #endif // TNT_FILAMENT_BACKEND_PRIVATE_SAMPLERGROUP_H diff --git a/filament/backend/src/metal/MetalHandles.h b/filament/backend/src/metal/MetalHandles.h index 52f04266f4..5da7d952dc 100644 --- a/filament/backend/src/metal/MetalHandles.h +++ b/filament/backend/src/metal/MetalHandles.h @@ -102,7 +102,7 @@ private: // These two fields store a callback and user data to notify the client that a frame is ready // for presentation. // If frameScheduledCallback is nullptr, then the Metal backend automatically calls - // presentDrawable when the frame is commited. + // presentDrawable when the frame is committed. // Otherwise, the Metal backend will not automatically present the frame. Instead, clients bear // the responsibility of presenting the frame by calling the PresentCallable object. FrameScheduledCallback frameScheduledCallback = nullptr; diff --git a/filament/backend/src/noop/NoopDriver.h b/filament/backend/src/noop/NoopDriver.h index ef1760e967..5b4037f657 100644 --- a/filament/backend/src/noop/NoopDriver.h +++ b/filament/backend/src/noop/NoopDriver.h @@ -35,6 +35,8 @@ public: private: ShaderModel getShaderModel() const noexcept final; + uint64_t nextFakeHandle = 1; + /* * Driver interface */ @@ -50,7 +52,7 @@ private: #define DECL_DRIVER_API_RETURN(RetType, methodName, paramsDecl, params) \ RetType methodName##S() noexcept override { \ - return RetType((RetType::HandleId)0xDEAD0000); } \ + return RetType((RetType::HandleId)nextFakeHandle++); } \ UTILS_ALWAYS_INLINE inline void methodName##R(RetType, paramsDecl) { } #include "private/backend/DriverAPI.inc" diff --git a/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm b/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm index 107c8a64b4..c495968936 100644 --- a/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm +++ b/filament/backend/src/opengl/platforms/PlatformCocoaGL.mm @@ -145,7 +145,6 @@ Driver* PlatformCocoaGL::createDriver(void* sharedContext) noexcept { NSOpenGLPFADepthSize, (NSOpenGLPixelFormatAttribute) 24, NSOpenGLPFADoubleBuffer, (NSOpenGLPixelFormatAttribute) true, NSOpenGLPFAAccelerated, (NSOpenGLPixelFormatAttribute) true, - NSOpenGLPFANoRecovery, (NSOpenGLPixelFormatAttribute) true, 0, 0, }; diff --git a/filament/backend/src/opengl/platforms/PlatformCocoaTouchGL.mm b/filament/backend/src/opengl/platforms/PlatformCocoaTouchGL.mm index 9ede3a6dca..0690ae5d18 100644 --- a/filament/backend/src/opengl/platforms/PlatformCocoaTouchGL.mm +++ b/filament/backend/src/opengl/platforms/PlatformCocoaTouchGL.mm @@ -42,6 +42,7 @@ using namespace backend; struct PlatformCocoaTouchGLImpl { EAGLContext* mGLContext = nullptr; CAEAGLLayer* mCurrentGlLayer = nullptr; + std::vector mHeadlessGlLayers; CGRect mCurrentGlLayerRect; GLuint mDefaultFramebuffer = 0; GLuint mDefaultColorbuffer = 0; @@ -108,11 +109,28 @@ Platform::SwapChain* PlatformCocoaTouchGL::createSwapChain(void* nativewindow, u } Platform::SwapChain* PlatformCocoaTouchGL::createSwapChain(uint32_t width, uint32_t height, uint64_t& flags) noexcept { - // TODO: implement headless SwapChain - return nullptr; + CAEAGLLayer* glLayer = [CAEAGLLayer layer]; + glLayer.frame = CGRectMake(0, 0, width, height); + + // adding the pointer to the array retains the CAEAGLLayer + pImpl->mHeadlessGlLayers.push_back(glLayer); + + return (__bridge SwapChain*) glLayer; } void PlatformCocoaTouchGL::destroySwapChain(Platform::SwapChain* swapChain) noexcept { + CAEAGLLayer* glLayer = (__bridge CAEAGLLayer*) swapChain; + + if (pImpl->mCurrentGlLayer == glLayer) { + pImpl->mCurrentGlLayer = nullptr; + } + + auto& v = pImpl->mHeadlessGlLayers; + auto it = std::find(v.begin(), v.end(), glLayer); + if(it != v.end()) { + // removing the pointer from the array releases the CAEAGLLayer + v.erase(it); + } } void PlatformCocoaTouchGL::createDefaultRenderTarget(uint32_t& framebuffer, uint32_t& colorbuffer, diff --git a/filament/backend/src/opengl/platforms/PlatformWGL.cpp b/filament/backend/src/opengl/platforms/PlatformWGL.cpp index 31fe637a8f..f0b836e7b8 100644 --- a/filament/backend/src/opengl/platforms/PlatformWGL.cpp +++ b/filament/backend/src/opengl/platforms/PlatformWGL.cpp @@ -103,7 +103,6 @@ Driver* PlatformWGL::createDriver(void* const sharedGLContext) noexcept { int attribs[] = { WGL_CONTEXT_MAJOR_VERSION_ARB, 4, WGL_CONTEXT_MINOR_VERSION_ARB, 1, - WGL_CONTEXT_FLAGS_ARB, WGL_CONTEXT_PROFILE_MASK_ARB, 0 }; diff --git a/filament/docs/Debugging.md b/filament/docs/Debugging.md index 3400bb828d..d0c7ffe500 100644 --- a/filament/docs/Debugging.md +++ b/filament/docs/Debugging.md @@ -2,7 +2,8 @@ ## Debugging Vulkan on macOS -See "Enabling Validation" in `third_party/moltenvk/README.md`. +Install the LunarG SDK and allow the installer to include "System Global Components", which should +be enabled by default. ## Debugging Vulkan on Linux diff --git a/filament/src/Allocators.h b/filament/src/Allocators.h index 25ccab736c..07a4641ee9 100644 --- a/filament/src/Allocators.h +++ b/filament/src/Allocators.h @@ -49,7 +49,8 @@ static constexpr size_t CONFIG_COMMAND_BUFFERS_SIZE = 3 * CONFIG_MIN_COMM using HeapAllocatorArena = utils::Arena< utils::HeapAllocator, utils::LockingPolicy::Mutex, - utils::TrackingPolicy::DebugAndHighWatermark>; + utils::TrackingPolicy::DebugAndHighWatermark, + utils::AreaPolicy::NullArea>; using LinearAllocatorArena = utils::Arena< utils::LinearAllocator, @@ -62,7 +63,9 @@ using LinearAllocatorArena = utils::Arena< // intrinsically synchronized as it relies on heap allocations (i.e.: malloc/free) using HeapAllocatorArena = utils::Arena< utils::HeapAllocator, - utils::LockingPolicy::NoLock>; + utils::LockingPolicy::NoLock, + utils::TrackingPolicy::Untracked, + utils::AreaPolicy::NullArea>; using LinearAllocatorArena = utils::Arena< utils::LinearAllocator, diff --git a/filament/src/HwRenderPrimitiveFactory.cpp b/filament/src/HwRenderPrimitiveFactory.cpp new file mode 100644 index 0000000000..2e996e2595 --- /dev/null +++ b/filament/src/HwRenderPrimitiveFactory.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HwRenderPrimitiveFactory.h" + +#include + +namespace filament { + +using namespace utils; +using namespace backend; + +bool operator<(HwRenderPrimitiveFactory::Key const& lhs, + HwRenderPrimitiveFactory::Key const& rhs) noexcept { + if (lhs.vbh == rhs.vbh) { + if (lhs.ibh == rhs.ibh) { + if (lhs.offset == rhs.offset) { + if (lhs.count == rhs.count) { + return lhs.type < rhs.type; + } else { + return lhs.count < rhs.count; + } + } else { + return lhs.offset < rhs.offset; + } + } else { + return lhs.ibh < rhs.ibh; + } + } else { + return lhs.vbh < rhs.vbh; + } +} + +inline bool operator<(HwRenderPrimitiveFactory::Entry const& lhs, + HwRenderPrimitiveFactory::Entry const& rhs) noexcept { + return lhs.key < rhs.key; +} + +inline bool operator<(HwRenderPrimitiveFactory::Key const& lhs, + HwRenderPrimitiveFactory::Entry const& rhs) noexcept { + return lhs < rhs.key; +} + +inline bool operator<(HwRenderPrimitiveFactory::Entry const& lhs, + HwRenderPrimitiveFactory::Key const& rhs) noexcept { + return lhs.key < rhs; +} + +// ------------------------------------------------------------------------------------------------ + +HwRenderPrimitiveFactory::HwRenderPrimitiveFactory() + : mArena("HwRenderPrimitiveFactory::mArena", SET_ARENA_SIZE), + mSet(mArena) { + mMap.reserve(256); +} + +HwRenderPrimitiveFactory::~HwRenderPrimitiveFactory() noexcept = default; + +void HwRenderPrimitiveFactory::terminate(DriverApi& driver) noexcept { + assert_invariant(mMap.empty()); + assert_invariant(mSet.empty()); +} + +RenderPrimitiveHandle HwRenderPrimitiveFactory::create(DriverApi& driver, + VertexBufferHandle vbh, IndexBufferHandle ibh, + PrimitiveType type, uint32_t offset, uint32_t minIndex, uint32_t maxIndex, + uint32_t count) noexcept { + + const Key key = { vbh, ibh, offset, count, type }; + + // see if we already have seen this RenderPrimitive + auto pos = mSet.find(key); + + // the common case is that we've never seen it (i.e.: no reuse) + if (UTILS_LIKELY(pos == mSet.end())) { + // create the backend object + auto handle = driver.createRenderPrimitive(vbh, ibh, + type, offset, minIndex, maxIndex, count); + // insert key/handle in our set with a refcount of 1 + // IMPORTANT: std::set<> doesn't invalidate iterators in insert/erase + auto [ipos, _] = mSet.insert({ key, handle, 1 }); + // map the handle back to the key/payload + mMap.insert({ handle.getId(), ipos }); + return handle; + } + pos->refs++; + return pos->handle; +} + +void HwRenderPrimitiveFactory::destroy(DriverApi& driver, RenderPrimitiveHandle rph) noexcept { + // look for this handle in our map + auto pos = mMap.find(rph.getId()); + + // it must be there + assert_invariant(pos != mMap.end()); + + // check the refcount and destroy if needed + auto ipos = pos->second; + if (--ipos->refs == 0) { + mSet.erase(ipos); + mMap.erase(pos); + driver.destroyRenderPrimitive(rph); + } +} + +} // namespace filament diff --git a/filament/src/HwRenderPrimitiveFactory.h b/filament/src/HwRenderPrimitiveFactory.h new file mode 100644 index 0000000000..f2f5bbb5ab --- /dev/null +++ b/filament/src/HwRenderPrimitiveFactory.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_HWRENDERPRIMITIVEFACTORY_H +#define TNT_FILAMENT_HWRENDERPRIMITIVEFACTORY_H + +#include +#include + +#include + +#include + +#include + +#include + +#include + +namespace filament { + +class FEngine; + +class HwRenderPrimitiveFactory { +public: + + HwRenderPrimitiveFactory(); + ~HwRenderPrimitiveFactory() noexcept; + + HwRenderPrimitiveFactory(HwRenderPrimitiveFactory const& rhs) = delete; + HwRenderPrimitiveFactory(HwRenderPrimitiveFactory&& rhs) noexcept = delete; + HwRenderPrimitiveFactory& operator=(HwRenderPrimitiveFactory const& rhs) = delete; + HwRenderPrimitiveFactory& operator=(HwRenderPrimitiveFactory&& rhs) noexcept = delete; + + void terminate(backend::DriverApi& driver) noexcept; + + backend::RenderPrimitiveHandle create(backend::DriverApi& driver, + backend::VertexBufferHandle vbh, + backend::IndexBufferHandle ibh, + backend::PrimitiveType type, + uint32_t offset, + uint32_t minIndex, + uint32_t maxIndex, + uint32_t count) noexcept; + + void destroy(backend::DriverApi& driver, + backend::RenderPrimitiveHandle rph) noexcept; + +private: + struct Key { // 20 bytes + backend::VertexBufferHandle vbh; // 4 + backend::IndexBufferHandle ibh; // 4 + uint32_t offset; // 4 + uint32_t count; // 4 + backend::PrimitiveType type; // 4 + }; + + struct Entry { // 28 bytes + Key key; // 20 + backend::RenderPrimitiveHandle handle; // 4 + mutable uint32_t refs; // 4 + }; + + + // the std::set node size is sizeof(std::set::node_type) + sizeof(T), aligned. + // we need this value for our pool allocator. On clang 64 bits this is 56 bytes. + using DummySetAllocator = utils::STLAllocator, utils::LockingPolicy::NoLock>>; + using DummySetNode = std::pair, DummySetAllocator>::node_type, Entry>; + + // Size of the arena used for the "set" part of the bimap + static constexpr size_t SET_ARENA_SIZE = 4 * 1024 * 1024; + + // Arena for the set<>, using a pool allocator inside a heap area. + using Arena = utils::Arena< + utils::PoolAllocator, + utils::LockingPolicy::NoLock, +#ifndef NDEBUG + utils::TrackingPolicy::HighWatermark, +#else + utils::TrackingPolicy::Untracked, +#endif + utils::AreaPolicy::HeapArea>; + + using Set = std::set< + Entry, + std::less<>, + utils::STLAllocator>; + + using Map = tsl::robin_map< + backend::RenderPrimitiveHandle::HandleId, + Set::const_iterator>; + + // Arena where the set memory is allocated + Arena mArena; + + // set of HwRenderPrimitive data + Set mSet; + + // map of RenderPrimitiveHandle to Set Entry + Map mMap; + + friend bool operator<(Key const& lhs, Key const& rhs) noexcept; + friend bool operator<(Key const& lhs, Entry const& rhs) noexcept; + friend bool operator<(Entry const& lhs, Key const& rhs) noexcept; + friend bool operator<(Entry const& lhs, Entry const& rhs) noexcept; +}; + +} // namespace filament + +#endif // TNT_FILAMENT_HWRENDERPRIMITIVEFACTORY_H diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h index be31ee191e..e0705178e1 100644 --- a/filament/src/PostProcessManager.h +++ b/filament/src/PostProcessManager.h @@ -316,7 +316,12 @@ private: mutable bool mHasMaterial{}; }; - tsl::robin_map mMaterialRegistry; + using MaterialRegistryMap = tsl::robin_map< + utils::StaticString, + PostProcessMaterial, + utils::StaticString::Hasher>; + + MaterialRegistryMap mMaterialRegistry; void registerPostProcessMaterial(utils::StaticString name, uint8_t const* data, int size); PostProcessMaterial& getPostProcessMaterial(utils::StaticString name) noexcept; diff --git a/filament/src/RenderPass.cpp b/filament/src/RenderPass.cpp index 91f210684d..e0291e7144 100644 --- a/filament/src/RenderPass.cpp +++ b/filament/src/RenderPass.cpp @@ -81,12 +81,18 @@ void RenderPass::setCamera(const CameraInfo& camera) noexcept { mCameraForwardVector = camera.getForwardVector(); } -void RenderPass::overridePolygonOffset(backend::PolygonOffset* polygonOffset) noexcept { +void RenderPass::overridePolygonOffset(backend::PolygonOffset const* polygonOffset) noexcept { if ((mPolygonOffsetOverride = (polygonOffset != nullptr))) { mPolygonOffset = *polygonOffset; } } +void RenderPass::overrideScissor(backend::Viewport const* scissor) noexcept { + if ((mScissorOverride = (scissor != nullptr))) { + mScissor = *scissor; + } +} + void RenderPass::appendCommands(CommandTypeFlags const commandTypeFlags) noexcept { SYSTRACE_CONTEXT(); @@ -580,10 +586,16 @@ void RenderPass::Executor::recordDriverCommands(FEngine& engine, backend::Driver if (first != last) { SYSTRACE_VALUE32("commandCount", last - first); - PolygonOffset dummyPolyOffset; - PipelineState pipeline{ .polygonOffset = mPolygonOffset }; - PolygonOffset* const pPipelinePolygonOffset = - mPolygonOffsetOverride ? &dummyPolyOffset : &pipeline.polygonOffset; + PipelineState pipeline{ + .polygonOffset = mPolygonOffset, + .scissor = mScissor + }, dummyPipeline; + + auto* const pPipelinePolygonOffset = + mPolygonOffsetOverride ? &dummyPipeline.polygonOffset : &pipeline.polygonOffset; + + auto* const pScissor = + mScissorOverride ? &dummyPipeline.scissor : &pipeline.scissor; Handle uboHandle = mUboHandle; FMaterialInstance const* UTILS_RESTRICT mi = nullptr; @@ -616,7 +628,7 @@ void RenderPass::Executor::recordDriverCommands(FEngine& engine, backend::Driver // this is always taken the first time mi = info.mi; ma = mi->getMaterial(); - pipeline.scissor = mi->getScissor(); + *pScissor = mi->getScissor(); *pPipelinePolygonOffset = mi->getPolygonOffset(); mi->use(driver); } @@ -657,7 +669,9 @@ RenderPass::Executor::Executor(RenderPass const* pass, Command const* b, Command : mEngine(pass->mEngine), mBegin(b), mEnd(e), mCustomCommands(pass->mCustomCommands), mUboHandle(pass->mUboHandle), mPolygonOffset(pass->mPolygonOffset), - mPolygonOffsetOverride(pass->mPolygonOffsetOverride) { + mScissor(pass->mScissor), + mPolygonOffsetOverride(pass->mPolygonOffsetOverride), + mScissorOverride(pass->mScissorOverride) { assert_invariant(b >= pass->begin()); assert_invariant(e <= pass->end()); } diff --git a/filament/src/RenderPass.h b/filament/src/RenderPass.h index b8b6801d43..d8d06ecf4c 100644 --- a/filament/src/RenderPass.h +++ b/filament/src/RenderPass.h @@ -276,7 +276,10 @@ public: ~RenderPass() noexcept; // if non-null, overrides the material's polygon offset - void overridePolygonOffset(backend::PolygonOffset* polygonOffset) noexcept; + void overridePolygonOffset(backend::PolygonOffset const* polygonOffset) noexcept; + + // if non-null, overrides the material's scissor + void overrideScissor(backend::Viewport const* scissor) noexcept; // specifies the geometry to generate commands for void setGeometry(FScene::RenderableSoa const& soa, utils::Range vr, @@ -330,7 +333,9 @@ public: const CustomCommandVector mCustomCommands; const backend::Handle mUboHandle; const backend::PolygonOffset mPolygonOffset; - const bool mPolygonOffsetOverride; + const backend::Viewport mScissor; + const bool mPolygonOffsetOverride : 1; + const bool mScissorOverride : 1; Executor(RenderPass const* pass, Command const* b, Command const* e) noexcept; @@ -430,9 +435,15 @@ private: // whether to override the polygon offset setting bool mPolygonOffsetOverride = false; + // whether to override the polygon offset setting + bool mScissorOverride = false; + // value of the override backend::PolygonOffset mPolygonOffset{}; + // value of scissor override + backend::Viewport mScissor{}; + // a vector for our custom commands mutable Executor::CustomCommandVector mCustomCommands; }; diff --git a/filament/src/RenderPrimitive.cpp b/filament/src/RenderPrimitive.cpp index a46fbc4c21..12ecb6c994 100644 --- a/filament/src/RenderPrimitive.cpp +++ b/filament/src/RenderPrimitive.cpp @@ -25,7 +25,7 @@ namespace filament { -void FRenderPrimitive::init(backend::DriverApi& driver, +void FRenderPrimitive::init(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, const RenderableManager::Builder::Entry& entry) noexcept { assert_invariant(entry.materialInstance); @@ -42,7 +42,7 @@ void FRenderPrimitive::init(backend::DriverApi& driver, auto const& ebh = vertexBuffer->getHwHandle(); auto const& ibh = indexBuffer->getHwHandle(); - mHandle = driver.createRenderPrimitive(ebh, ibh, entry.type, (uint32_t)entry.offset, + mHandle = factory.create(driver, ebh, ibh, entry.type, (uint32_t)entry.offset, (uint32_t)entry.minIndex, (uint32_t)entry.maxIndex, (uint32_t)entry.count); mPrimitiveType = entry.type; @@ -50,14 +50,14 @@ void FRenderPrimitive::init(backend::DriverApi& driver, } } -void FRenderPrimitive::terminate(FEngine& engine) { - FEngine::DriverApi& driver = engine.getDriverApi(); +void FRenderPrimitive::terminate(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver) { if (mHandle) { - driver.destroyRenderPrimitive(mHandle); + factory.destroy(driver, mHandle); } } -void FRenderPrimitive::set(FEngine& engine, RenderableManager::PrimitiveType type, +void FRenderPrimitive::set(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, + RenderableManager::PrimitiveType type, FVertexBuffer* vertices, FIndexBuffer* indices, size_t offset, size_t minIndex, size_t maxIndex, size_t count) noexcept { @@ -66,13 +66,11 @@ void FRenderPrimitive::set(FEngine& engine, RenderableManager::PrimitiveType typ auto const& ebh = vertices->getHwHandle(); auto const& ibh = indices->getHwHandle(); - FEngine::DriverApi& driver = engine.getDriverApi(); - if (mHandle) { - driver.destroyRenderPrimitive(mHandle); + factory.destroy(driver, mHandle); } - mHandle = driver.createRenderPrimitive(ebh, ibh, type, + mHandle = factory.create(driver, ebh, ibh, type, (uint32_t)offset, (uint32_t)minIndex, (uint32_t)maxIndex, (uint32_t)count); mPrimitiveType = type; diff --git a/filament/src/RenderPrimitive.h b/filament/src/RenderPrimitive.h index d8afc78561..91d4f42377 100644 --- a/filament/src/RenderPrimitive.h +++ b/filament/src/RenderPrimitive.h @@ -31,19 +31,22 @@ class FEngine; class FVertexBuffer; class FIndexBuffer; class FRenderer; +class HwRenderPrimitiveFactory; class FRenderPrimitive { public: FRenderPrimitive() noexcept = default; - void init(backend::DriverApi& driver, const RenderableManager::Builder::Entry& entry) noexcept; + void init(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, + const RenderableManager::Builder::Entry& entry) noexcept; - void set(FEngine& engine, RenderableManager::PrimitiveType type, + void set(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, + RenderableManager::PrimitiveType type, FVertexBuffer* vertices, FIndexBuffer* indices, size_t offset, size_t minIndex, size_t maxIndex, size_t count) noexcept; // frees driver resources, object becomes invalid - void terminate(FEngine& engine); + void terminate(HwRenderPrimitiveFactory& factory, backend::DriverApi& driver); const FMaterialInstance* getMaterialInstance() const noexcept { return mMaterialInstance; } backend::Handle getHwHandle() const noexcept { return mHandle; } diff --git a/filament/src/ResourceAllocator.cpp b/filament/src/ResourceAllocator.cpp index 02b9412ad9..319955927d 100644 --- a/filament/src/ResourceAllocator.cpp +++ b/filament/src/ResourceAllocator.cpp @@ -34,6 +34,17 @@ using namespace backend; // ------------------------------------------------------------------------------------------------ +template +UTILS_NOINLINE +ResourceAllocator::AssociativeContainer::AssociativeContainer() { + mContainer.reserve(128); +} + +template +UTILS_NOINLINE +ResourceAllocator::AssociativeContainer::~AssociativeContainer() noexcept { +} + template UTILS_NOINLINE typename ResourceAllocator::AssociativeContainer::iterator diff --git a/filament/src/ResourceAllocator.h b/filament/src/ResourceAllocator.h index 7462cdd4b3..9ad1c89130 100644 --- a/filament/src/ResourceAllocator.h +++ b/filament/src/ResourceAllocator.h @@ -153,8 +153,7 @@ private: template struct Hasher> { std::size_t operator()(backend::Handle const& s) const noexcept { - std::hash::HandleId> hash{}; - return hash(s.getId()); + return s.getId(); } }; @@ -163,11 +162,14 @@ private: template> class AssociativeContainer { // We use a std::vector instead of a std::multimap because we don't expect many items - // in the cache and std::multimap generates tons of code. Even with more items, we - // could improve this trivially by using a sorted std::vector. + // in the cache and std::multimap generates tons of code. std::multimap starts getting + // significantly better around 1000 items. using Container = std::vector>; Container mContainer; + public: + AssociativeContainer(); + ~AssociativeContainer() noexcept; using iterator = typename Container::iterator; using const_iterator = typename Container::const_iterator; using key_type = typename Container::value_type::first_type; @@ -186,12 +188,13 @@ private: }; using CacheContainer = AssociativeContainer; + using InUseContainer = AssociativeContainer; CacheContainer::iterator purge(CacheContainer::iterator const& pos); backend::DriverApi& mBackend; CacheContainer mTextureCache; - AssociativeContainer mInUseTextures; + InUseContainer mInUseTextures; size_t mAge = 0; uint32_t mCacheSize = 0; static constexpr bool mEnabled = true; diff --git a/filament/src/ShadowMap.cpp b/filament/src/ShadowMap.cpp index 405f614c91..21f1a9e401 100644 --- a/filament/src/ShadowMap.cpp +++ b/filament/src/ShadowMap.cpp @@ -74,9 +74,13 @@ ShadowMap::~ShadowMap() { void ShadowMap::render(FScene const& scene, utils::Range range, FScene::VisibleMaskType visibilityMask, RenderPass* const pass) noexcept { + const backend::Viewport disabledScissor{ 0, 0, + (uint32_t)std::numeric_limits::max(), + (uint32_t)std::numeric_limits::max() }; pass->setVisibilityMask(visibilityMask); pass->setGeometry(scene.getRenderableData(), range, scene.getRenderableUBO()); pass->overridePolygonOffset(&mShadowMapInfo.polygonOffset); + pass->overrideScissor(&disabledScissor); pass->appendCommands(RenderPass::SHADOW); pass->sortCommands(); } diff --git a/filament/src/components/RenderableManager.cpp b/filament/src/components/RenderableManager.cpp index 45373f7159..058f6a027b 100644 --- a/filament/src/components/RenderableManager.cpp +++ b/filament/src/components/RenderableManager.cpp @@ -334,8 +334,9 @@ void FRenderableManager::create( Builder::Entry const * const entries = builder->mEntries.data(); const size_t entryCount = builder->mEntries.size(); FRenderPrimitive* rp = new FRenderPrimitive[entryCount]; + auto& factory = mHwRenderPrimitiveFactory; for (size_t i = 0; i < entryCount; ++i) { - rp[i].init(driver, entries[i]); + rp[i].init(factory, driver, entries[i]); } setPrimitives(ci, { rp, size_type(entryCount) }); @@ -463,6 +464,7 @@ void FRenderableManager::terminate() noexcept { manager.removeComponent(manager.getEntity(ci)); } } + mHwRenderPrimitiveFactory.terminate(mEngine.getDriverApi()); } void FRenderableManager::gc(utils::EntityManager& em) noexcept { @@ -477,7 +479,7 @@ void FRenderableManager::destroyComponent(Instance ci) noexcept { FEngine::DriverApi& driver = engine.getDriverApi(); // See create(RenderableManager::Builder&, Entity) - destroyComponentPrimitives(engine, manager[ci].primitives); + destroyComponentPrimitives(mHwRenderPrimitiveFactory, driver, manager[ci].primitives); destroyComponentMorphTargets(engine, manager[ci].morphTargets); // destroy the bones structures if any @@ -494,9 +496,10 @@ void FRenderableManager::destroyComponent(Instance ci) noexcept { } void FRenderableManager::destroyComponentPrimitives( - FEngine& engine, Slice& primitives) noexcept { + HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, + Slice& primitives) noexcept { for (auto& primitive : primitives) { - primitive.terminate(engine); + primitive.terminate(factory, driver); } delete[] primitives.data(); } @@ -573,8 +576,8 @@ void FRenderableManager::setGeometryAt(Instance instance, uint8_t level, size_t if (instance) { Slice& primitives = getRenderPrimitives(instance, level); if (primitiveIndex < primitives.size()) { - primitives[primitiveIndex].set(mEngine, type, vertices, indices, offset, - 0, vertices->getVertexCount() - 1, count); + primitives[primitiveIndex].set(mHwRenderPrimitiveFactory, mEngine.getDriverApi(), + type, vertices, indices, offset, 0, vertices->getVertexCount() - 1, count); } } } diff --git a/filament/src/components/RenderableManager.h b/filament/src/components/RenderableManager.h index 326360b67c..78c613d7ad 100644 --- a/filament/src/components/RenderableManager.h +++ b/filament/src/components/RenderableManager.h @@ -19,6 +19,7 @@ #include "upcast.h" +#include "HwRenderPrimitiveFactory.h" #include "UniformBuffer.h" #include "private/backend/DriverApiForward.h" @@ -175,7 +176,8 @@ public: private: void destroyComponent(Instance ci) noexcept; - static void destroyComponentPrimitives(FEngine& engine, + static void destroyComponentPrimitives( + HwRenderPrimitiveFactory& factory, backend::DriverApi& driver, utils::Slice& primitives) noexcept; static void destroyComponentMorphTargets(FEngine& engine, utils::Slice& morphTargets) noexcept; @@ -252,6 +254,7 @@ private: Sim mManager; FEngine& mEngine; + HwRenderPrimitiveFactory mHwRenderPrimitiveFactory; }; FILAMENT_UPCAST(RenderableManager) diff --git a/filament/src/details/DebugRegistry.h b/filament/src/details/DebugRegistry.h index 017e6e4fb5..5207629511 100644 --- a/filament/src/details/DebugRegistry.h +++ b/filament/src/details/DebugRegistry.h @@ -72,8 +72,8 @@ private: bool hasProperty(const char* name) const noexcept; void* getPropertyAddress(const char* name) noexcept; DataSource getDataSource(const char* name) const noexcept; - std::unordered_map mPropertyMap; - std::unordered_map mDataSourceMap; + std::unordered_map mPropertyMap; + std::unordered_map mDataSourceMap; }; FILAMENT_UPCAST(DebugRegistry) diff --git a/filament/src/details/Engine.cpp b/filament/src/details/Engine.cpp index 04eb26ae87..8a462c49eb 100644 --- a/filament/src/details/Engine.cpp +++ b/filament/src/details/Engine.cpp @@ -178,7 +178,8 @@ FEngine::FEngine(Backend backend, Platform* platform, void* sharedGLContext) : mLightManager(*this), mCameraManager(*this), mCommandBufferQueue(CONFIG_MIN_COMMAND_BUFFERS_SIZE, CONFIG_COMMAND_BUFFERS_SIZE), - mPerRenderPassAllocator("per-renderpass allocator", CONFIG_PER_RENDER_PASS_ARENA_SIZE), + mPerRenderPassAllocator("FEngine::mPerRenderPassAllocator", CONFIG_PER_RENDER_PASS_ARENA_SIZE), + mHeapAllocator("FEngine::mHeapAllocator", AreaPolicy::NullArea{}), mJobSystem(getJobSystemThreadPoolSize()), mEngineEpoch(std::chrono::steady_clock::now()), mDriverBarrier(1), diff --git a/filament/src/details/Scene.h b/filament/src/details/Scene.h index bd79755b2a..b578d32fcb 100644 --- a/filament/src/details/Scene.h +++ b/filament/src/details/Scene.h @@ -208,7 +208,7 @@ private: * (a vector<> could work, but removes would be O(n)). robin_set<> iterates almost as * nicely as vector<>, which is a good compromise. */ - tsl::robin_set mEntities; + tsl::robin_set mEntities; /* diff --git a/filament/src/fg/Blackboard.h b/filament/src/fg/Blackboard.h index 0912c0f893..808ba08c40 100644 --- a/filament/src/fg/Blackboard.h +++ b/filament/src/fg/Blackboard.h @@ -26,7 +26,10 @@ namespace filament { class Blackboard { - using Container = std::unordered_map; + using Container = std::unordered_map< + utils::StaticString, + FrameGraphHandle, + utils::StaticString::Hasher>; public: Blackboard() noexcept; diff --git a/ios/CocoaPods/Filament.podspec b/ios/CocoaPods/Filament.podspec index 6865e27409..a80d9c0241 100644 --- a/ios/CocoaPods/Filament.podspec +++ b/ios/CocoaPods/Filament.podspec @@ -1,12 +1,12 @@ Pod::Spec.new do |spec| spec.name = "Filament" - spec.version = "1.23.1" + spec.version = "1.23.2" spec.license = { :type => "Apache 2.0", :file => "LICENSE" } spec.homepage = "https://google.github.io/filament" spec.authors = "Google LLC." spec.summary = "Filament is a real-time physically based rendering engine for Android, iOS, Windows, Linux, macOS, and WASM/WebGL." spec.platform = :ios, "11.0" - spec.source = { :http => "https://github.com/google/filament/releases/download/v1.23.1/filament-v1.23.1-ios.tgz" } + spec.source = { :http => "https://github.com/google/filament/releases/download/v1.23.2/filament-v1.23.2-ios.tgz" } # Fix linking error with Xcode 12; we do not yet support the simulator on Apple silicon. spec.pod_target_xcconfig = { diff --git a/ios/samples/gltf-viewer/gltf-viewer.xcodeproj/project.pbxproj b/ios/samples/gltf-viewer/gltf-viewer.xcodeproj/project.pbxproj index cf241bf27a..370a6b584e 100644 --- a/ios/samples/gltf-viewer/gltf-viewer.xcodeproj/project.pbxproj +++ b/ios/samples/gltf-viewer/gltf-viewer.xcodeproj/project.pbxproj @@ -432,6 +432,7 @@ "-lviewer", "-lcivetweb", "-lktxreader", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.gltf-viewer"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -487,6 +488,7 @@ "-lviewer", "-lcivetweb", "-lktxreader", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.gltf-viewer"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -604,6 +606,7 @@ "-lviewer", "-lcivetweb", "-lktxreader", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.gltf-viewer"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -659,6 +662,7 @@ "-lviewer", "-lcivetweb", "-lktxreader", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.gltf-viewer"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; diff --git a/ios/samples/gltf-viewer/project.yml b/ios/samples/gltf-viewer/project.yml index afc744ed72..36076af9d1 100644 --- a/ios/samples/gltf-viewer/project.yml +++ b/ios/samples/gltf-viewer/project.yml @@ -20,7 +20,7 @@ targets: settings: base: OTHER_LDFLAGS: ["-lgltfio_core", "-lgltfio_resources", "-limage", "-lgeometry", "-lstb", "-lbasis_transcoder", - "-lzstd", "-lcamutils", "-ldracodec", "-lviewer", "-lcivetweb", "-lktxreader"] + "-lzstd", "-lcamutils", "-ldracodec", "-lviewer", "-lcivetweb", "-lktxreader", "-luberzlib"] preBuildScripts: - path: build-resources.sh name: Build Resources diff --git a/ios/samples/hello-gltf/hello-gltf.xcodeproj/project.pbxproj b/ios/samples/hello-gltf/hello-gltf.xcodeproj/project.pbxproj index b7f7fd0079..e03f6782ac 100644 --- a/ios/samples/hello-gltf/hello-gltf.xcodeproj/project.pbxproj +++ b/ios/samples/hello-gltf/hello-gltf.xcodeproj/project.pbxproj @@ -138,7 +138,7 @@ }; }; buildConfigurationList = C7514DDCFDA3C5F803E239F9 /* Build configuration list for PBXProject "hello-gltf" */; - compatibilityVersion = "Xcode 11.0"; + compatibilityVersion = "Xcode 10.0"; developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( @@ -330,6 +330,8 @@ "-ldracodec", "-lktxreader", "-lstb", + "-lzstd", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.hello-gltf"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -380,6 +382,8 @@ "-ldracodec", "-lktxreader", "-lstb", + "-lzstd", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.hello-gltf"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -602,6 +606,8 @@ "-ldracodec", "-lktxreader", "-lstb", + "-lzstd", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.hello-gltf"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; @@ -652,6 +658,8 @@ "-ldracodec", "-lktxreader", "-lstb", + "-lzstd", + "-luberzlib", ); PRODUCT_BUNDLE_IDENTIFIER = "${SAMPLE_CODE_DISAMBIGUATOR}.google.filament.hello-gltf"; SAMPLE_CODE_DISAMBIGUATOR = "${DEVELOPMENT_TEAM}"; diff --git a/ios/samples/hello-gltf/hello-gltf.xcodeproj/xcshareddata/xcschemes/hello-gltf Metal.xcscheme b/ios/samples/hello-gltf/hello-gltf.xcodeproj/xcshareddata/xcschemes/hello-gltf Metal.xcscheme index d025d88ecb..71cdb5979f 100644 --- a/ios/samples/hello-gltf/hello-gltf.xcodeproj/xcshareddata/xcschemes/hello-gltf Metal.xcscheme +++ b/ios/samples/hello-gltf/hello-gltf.xcodeproj/xcshareddata/xcschemes/hello-gltf Metal.xcscheme @@ -1,7 +1,7 @@ + version = "1.3"> + version = "1.3"> ${DUMMY_SRC}) add_library(gltfio_resources ${DUMMY_SRC} ${RESGEN_SOURCE}) -# ================================================================================================== -# Build "lite" resources library, which contains only a few materials -# ================================================================================================== - -set(LITE_DIR ${CMAKE_CURRENT_BINARY_DIR}/lite) - -function(generate_lite_mat BLENDING) - set(DOUBLESIDED false) - set(TRANSPARENCY default) - set(SHADINGMODEL lit) - set(GENERATED_MAT "${LITE_DIR}/${SHADINGMODEL}_${BLENDING}.mat") - configure_file(materials/gltflite.mat.in ${GENERATED_MAT}) - set(MATERIAL_SRCS ${MATERIAL_SRCS} ${GENERATED_MAT} PARENT_SCOPE) -endfunction() - -set(MATERIAL_SRCS) -generate_lite_mat(fade) -generate_lite_mat(opaque) - -foreach (input_path ${MATERIAL_SRCS}) - get_filename_component(basename "${input_path}" NAME_WE) - set(output_path "${LITE_DIR}/${basename}.filamat") - add_custom_command( - OUTPUT ${output_path} - COMMAND matc ${MATC_BASE_FLAGS} -o ${output_path} ${input_path} - MAIN_DEPENDENCY ${input_path} - DEPENDS matc - COMMENT "Compiling material ${input_path} to ${output_path}" - ) -endforeach() - -set(RESOURCE_BINS - "${LITE_DIR}/lit_opaque.filamat" - "${LITE_DIR}/lit_fade.filamat") - -get_resgen_vars(${LITE_DIR} gltfresources_lite) - -add_custom_command( - OUTPUT ${RESGEN_OUTPUTS} - COMMAND resgen ${RESGEN_FLAGS} ${RESOURCE_BINS} - DEPENDS resgen ${RESOURCE_BINS} -) - -if (DEFINED RESGEN_SOURCE_FLAGS) - set_source_files_properties(${RESGEN_SOURCE} PROPERTIES COMPILE_FLAGS ${RESGEN_SOURCE_FLAGS}) -endif() - -set(DUMMY_SRC "${LITE_DIR}/dummy.c") -add_custom_command(OUTPUT ${DUMMY_SRC} COMMAND echo "//" > ${DUMMY_SRC}) - -add_library(gltfio_resources_lite ${DUMMY_SRC} ${RESGEN_SOURCE}) - # ================================================================================================== # Build the "core" library (no filamat) # ================================================================================================== include_directories(${PUBLIC_HDR_DIR} ${RESOURCE_DIR}) -link_libraries(math utils filament cgltf stb ktxreader geometry gltfio_resources tsl trie) +link_libraries(math utils filament cgltf stb ktxreader geometry gltfio_resources tsl trie uberzlib) add_library(gltfio_core STATIC ${PUBLIC_HDRS} ${SRCS}) @@ -200,16 +174,15 @@ if (NOT WEBGL AND NOT ANDROID AND NOT IOS) # ================================================================================================== # Installation # ================================================================================================== - install(TARGETS ${TARGET} gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) + install(TARGETS ${TARGET} gltfio_core gltfio_resources ARCHIVE DESTINATION lib/${DIST_DIR}) install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio DESTINATION include) else() - install(TARGETS gltfio_core gltfio_resources gltfio_resources_lite ARCHIVE DESTINATION lib/${DIST_DIR}) + install(TARGETS gltfio_core gltfio_resources ARCHIVE DESTINATION lib/${DIST_DIR}) # The Image.h header is only needed within Filament and does not need to be distributed. install(DIRECTORY ${PUBLIC_HDR_DIR}/gltfio DESTINATION include PATTERN "Image.h" EXCLUDE) install(FILES ${RESOURCE_DIR}/gltfresources.h DESTINATION include/gltfio/resources) - install(FILES ${LITE_DIR}/gltfresources_lite.h DESTINATION include/gltfio/resources) endif() diff --git a/libs/gltfio/README.md b/libs/gltfio/README.md new file mode 100644 index 0000000000..7f97ab2bb7 --- /dev/null +++ b/libs/gltfio/README.md @@ -0,0 +1,26 @@ +# Description + +`gltfio` is a loader library that consumes `gltf` or `glb` content and produces Filament +objects. For usage details, see the docstring for `AssetLoader`. + +gltfio has two plug-in interfaces, `TextureProvider` and `MaterialProvider`. Filament ships with +several ready-to-go implementations described below. + +- `MaterialProvider` creates Filament materials in response to certain glTF requirements. + - [UbershaderLoader](#ubershaderloader) loads pre-built materials. + - `MaterialGenerator` builds materials at run time using the `filamat` library. +- `TextureProvider` creates and populates Filament `Texture` objects. + - `StbProvider` uses the STB library to read PNG and JPEG files. + - `Ktx2Provider` uses the BasisU library to read KTX2 files. + +# UbershaderLoader + +`UbershaderLoader` is a ready-to-go implementation of the `MaterialProvider` interface that should +be used in applications that need fast startup times. There is no material compilation that +occurs at run time, but the shaders might be relatively large and complex. + +At load time, the ubershader loader consumes an *ubershader archive* which is a precompiled set of +materials bundled with formal descriptions of the glTF features that they support. + +The `uberz` command line tool consumes a list of `.spec` and `.filamat` files and produces a single +`.uberz` file. For details on these two file formats, see the README in `libs/uberz`. diff --git a/libs/gltfio/materials/ubershader.mat.in b/libs/gltfio/materials/base.mat.in similarity index 97% rename from libs/gltfio/materials/ubershader.mat.in rename to libs/gltfio/materials/base.mat.in index cfc0601d62..7ea8c737b1 100644 --- a/libs/gltfio/materials/ubershader.mat.in +++ b/libs/gltfio/materials/base.mat.in @@ -1,5 +1,5 @@ material { - name : ubershader_${SHADINGMODEL}_${BLENDING}, + name : base_${SHADINGMODEL}_${BLENDING}, requires : [ uv0, uv1, color ], shadingModel : ${SHADINGMODEL}, blending : ${BLENDING}, @@ -63,9 +63,17 @@ material { // Reflectance { type : float, name : reflectance } + + ${CUSTOM_PARAMS} ], } +vertex { + void materialVertex(inout MaterialVertexInputs material) { + ${CUSTOM_VERTEX} + } +} + fragment { void material(inout MaterialInputs material) { highp float2 uvs[2]; @@ -164,5 +172,7 @@ fragment { material.emissive.rgb *= texture(materialParams_emissiveMap, uv).rgb; } #endif + + ${CUSTOM_FRAGMENT} } } diff --git a/libs/gltfio/materials/base.spec.in b/libs/gltfio/materials/base.spec.in new file mode 100644 index 0000000000..6e6e9375f5 --- /dev/null +++ b/libs/gltfio/materials/base.spec.in @@ -0,0 +1,32 @@ +# Required Aspects +ShadingModel = ${SHADINGMODEL} +BlendingMode = ${BLENDING} + +# Core Features +VertexColors = optional +BaseColorTexture = optional +NormalTexture = optional +OcclusionTexture = optional +EmissiveTexture = optional +MetallicRoughnessTexture = optional +TextureTransforms = optional + +# ClearCoat Features +ClearCoat = optional +ClearCoatTexture = optional +ClearCoatRoughnessTexture = optional +ClearCoatNormalTexture = optional + +# Sheen Features +Sheen = unsupported +SheenColorTexture = unsupported +SheenRoughnessTexture = unsupported + +# Transmission and IOR Features +Transmission = unsupported +Ior = unsupported +TransmissionTexture = unsupported + +# Volume Features +Volume = unsupported +VolumeThicknessTexture = unsupported diff --git a/libs/gltfio/materials/gltflite.mat.in b/libs/gltfio/materials/gltflite.mat.in deleted file mode 100644 index ef91f582b8..0000000000 --- a/libs/gltfio/materials/gltflite.mat.in +++ /dev/null @@ -1,101 +0,0 @@ -material { - name : gltflite_${SHADINGMODEL}_${BLENDING}, - requires : [ uv0, uv1, color ], - shadingModel : ${SHADINGMODEL}, - blending : ${BLENDING}, - doubleSided : ${DOUBLESIDED}, - transparency : ${TRANSPARENCY}, - flipUV : false, - specularAmbientOcclusion : simple, - specularAntiAliasing : true, - reflections : screenspace, - parameters : [ - - // Base Color - { type : int, name : baseColorIndex }, - { type : float4, name : baseColorFactor }, - { type : sampler2d, name : baseColorMap }, - - // Metallic-Roughness Map - { type : int, name : metallicRoughnessIndex }, - { type : float, name : metallicFactor }, - { type : float, name : roughnessFactor }, - { type : sampler2d, name : metallicRoughnessMap }, - - // Normal Map - { type : int, name : normalIndex }, - { type : float, name : normalScale }, - { type : sampler2d, name : normalMap }, - - // Ambient Occlusion - { type : int, name : aoIndex }, - { type : float, name : aoStrength }, - { type : sampler2d, name : occlusionMap }, - - // Emissive Map - { type : int, name : emissiveIndex }, - { type : float3, name : emissiveFactor }, - { type : sampler2d, name : emissiveMap }, - - // Reflectance - { type : float, name : reflectance } - ], -} - -fragment { - void material(inout MaterialInputs material) { - float2 uvs[2]; - uvs[0] = getUV0(); - uvs[1] = getUV1(); - - #if !defined(SHADING_MODEL_UNLIT) - if (materialParams.normalIndex > -1) { - float2 uv = uvs[materialParams.normalIndex]; - material.normal = texture(materialParams_normalMap, uv).xyz * 2.0 - 1.0; - material.normal.xy *= materialParams.normalScale; - } - #endif - - prepareMaterial(material); - material.baseColor = materialParams.baseColorFactor; - - if (materialParams.baseColorIndex > -1) { - float2 uv = uvs[materialParams.baseColorIndex]; - material.baseColor *= texture(materialParams_baseColorMap, uv); - } - - #if defined(BLEND_MODE_TRANSPARENT) - material.baseColor.rgb *= material.baseColor.a; - #endif - - material.baseColor *= getColor(); - - #if !defined(SHADING_MODEL_UNLIT) - #if defined(SHADING_MODEL_LIT) - material.roughness = materialParams.roughnessFactor; - material.metallic = materialParams.metallicFactor; - material.reflectance = materialParams.reflectance; - #endif - - material.emissive = vec4(materialParams.emissiveFactor.rgb, 0.0); - - if (materialParams.metallicRoughnessIndex > -1) { - float2 uv = uvs[materialParams.metallicRoughnessIndex]; - - vec4 mr = texture(materialParams_metallicRoughnessMap, uv); - material.roughness *= mr.g; - material.metallic *= mr.b; - } - - if (materialParams.aoIndex > -1) { - float2 uv = uvs[materialParams.aoIndex]; - material.ambientOcclusion = texture(materialParams_occlusionMap, uv).r * - materialParams.aoStrength; - } - if (materialParams.emissiveIndex > -1) { - float2 uv = uvs[materialParams.emissiveIndex]; - material.emissive.rgb *= texture(materialParams_emissiveMap, uv).rgb; - } - #endif - } -} diff --git a/libs/gltfio/materials/sheen.mat.in b/libs/gltfio/materials/sheen.mat.in index 9cc191f7d7..f579f12ad8 100644 --- a/libs/gltfio/materials/sheen.mat.in +++ b/libs/gltfio/materials/sheen.mat.in @@ -1,11 +1,11 @@ material { - name : sheen_${SHADINGMODEL}_${BLENDING}, + name : sheen_ubershader, requires : [ uv0, uv1, color ], - shadingModel : ${SHADINGMODEL}, + shadingModel : lit, blending : opaque, depthWrite : true, - doubleSided : ${DOUBLESIDED}, - transparency : ${TRANSPARENCY}, + doubleSided : false, + transparency : default, flipUV : false, specularAmbientOcclusion : simple, specularAntiAliasing : true, @@ -56,9 +56,17 @@ material { // Reflectance { type : float, name : reflectance } + + ${CUSTOM_PARAMS} ], } +vertex { + void materialVertex(inout MaterialVertexInputs material) { + ${CUSTOM_VERTEX} + } +} + fragment { void material(inout MaterialInputs material) { highp float2 uvs[2]; @@ -127,5 +135,7 @@ fragment { uv = (vec3(uv, 1.0) * materialParams.emissiveUvMatrix).xy; material.emissive.rgb *= texture(materialParams_emissiveMap, uv).rgb; } + + ${CUSTOM_FRAGMENT} } } diff --git a/libs/gltfio/materials/sheen.spec.in b/libs/gltfio/materials/sheen.spec.in new file mode 100644 index 0000000000..dcd41bf25c --- /dev/null +++ b/libs/gltfio/materials/sheen.spec.in @@ -0,0 +1,19 @@ +ShadingModel=lit +BlendingMode=opaque + +Transmission=unsupported +Volume=unsupported +Ior=unsupported +ClearCoat=unsupported + +VertexColors=optional +BaseColorTexture=optional +NormalTexture=optional +OcclusionTexture=optional +EmissiveTexture=optional +MetallicRoughnessTexture=optional +TextureTransforms=optional + +Sheen=optional +SheenColorTexture=optional +SheenRoughnessTexture=optional diff --git a/libs/gltfio/materials/transmission.mat.in b/libs/gltfio/materials/transmission.mat.in index 6962540afb..d0b518631a 100644 --- a/libs/gltfio/materials/transmission.mat.in +++ b/libs/gltfio/materials/transmission.mat.in @@ -1,10 +1,10 @@ material { - name : transmission_${SHADINGMODEL}_${BLENDING}, + name : transmission_ubershader, requires : [ uv0, uv1, color ], - shadingModel : ${SHADINGMODEL}, + shadingModel : lit, blending : masked, - doubleSided : ${DOUBLESIDED}, - transparency : ${TRANSPARENCY}, + doubleSided : false, + transparency : default, flipUV : false, specularAmbientOcclusion : simple, specularAntiAliasing : true, @@ -53,9 +53,17 @@ material { // IOR { type : float, name : ior } + + ${CUSTOM_PARAMS} ], } +vertex { + void materialVertex(inout MaterialVertexInputs material) { + ${CUSTOM_VERTEX} + } +} + fragment { void material(inout MaterialInputs material) { highp float2 uvs[2]; @@ -117,5 +125,7 @@ fragment { uv = (vec3(uv, 1.0) * materialParams.emissiveUvMatrix).xy; material.emissive.rgb *= texture(materialParams_emissiveMap, uv).rgb; } + + ${CUSTOM_FRAGMENT} } } diff --git a/libs/gltfio/materials/transmission.spec.in b/libs/gltfio/materials/transmission.spec.in new file mode 100644 index 0000000000..9c1062631f --- /dev/null +++ b/libs/gltfio/materials/transmission.spec.in @@ -0,0 +1,22 @@ +ShadingModel=lit + +# Even though the transmission material actually has its +# blend mode set to "masked", we want the glTF loader to +# think that it is "opaque". +BlendingMode=opaque + +Sheen=unsupported +Volume=unsupported +ClearCoat=unsupported + +VertexColors=optional +BaseColorTexture=optional +NormalTexture=optional +OcclusionTexture=optional +EmissiveTexture=optional +MetallicRoughnessTexture=optional +TextureTransforms=optional + +Transmission=optional +Ior=optional +TransmissionTexture=optional diff --git a/libs/gltfio/materials/volume.mat.in b/libs/gltfio/materials/volume.mat.in index 2864da34a9..dd97e68a3d 100644 --- a/libs/gltfio/materials/volume.mat.in +++ b/libs/gltfio/materials/volume.mat.in @@ -1,10 +1,10 @@ material { - name : volume_${SHADINGMODEL}_${BLENDING}, + name : volume_ubershader, requires : [ uv0, uv1, color ], - shadingModel : ${SHADINGMODEL}, + shadingModel : lit, blending : masked, - doubleSided : ${DOUBLESIDED}, - transparency : ${TRANSPARENCY}, + doubleSided : false, + transparency : default, flipUV : false, specularAmbientOcclusion : simple, specularAntiAliasing : true, @@ -60,9 +60,18 @@ material { // IOR { type : float, name : ior } + + + ${CUSTOM_PARAMS} ], } +vertex { + void materialVertex(inout MaterialVertexInputs material) { + ${CUSTOM_VERTEX} + } +} + fragment { void material(inout MaterialInputs material) { highp float2 uvs[2]; @@ -132,5 +141,7 @@ fragment { uv = (vec3(uv, 1.0) * materialParams.emissiveUvMatrix).xy; material.emissive.rgb *= texture(materialParams_emissiveMap, uv).rgb; } + + ${CUSTOM_FRAGMENT} } } diff --git a/libs/gltfio/materials/volume.spec.in b/libs/gltfio/materials/volume.spec.in new file mode 100644 index 0000000000..d77b8148d3 --- /dev/null +++ b/libs/gltfio/materials/volume.spec.in @@ -0,0 +1,30 @@ +ShadingModel=lit + +# NOTE: this intentionally omits the blending mode spec because +# Filament requires MASKED blending for KHR_materials_volume. + +# Core Features +VertexColors=optional +BaseColorTexture=optional +NormalTexture=optional +OcclusionTexture=optional +EmissiveTexture=optional +MetallicRoughnessTexture=optional +TextureTransforms=optional + +# ClearCoat Features +ClearCoat=unsupported + +# Sheen Features +Sheen=unsupported +SheenColorTexture=unsupported +SheenRoughnessTexture=unsupported + +# Transmission and IOR Features +Transmission=optional +Ior=optional +TransmissionTexture=optional + +# Volume Features +Volume=optional +VolumeThicknessTexture=optional diff --git a/libs/gltfio/src/DependencyGraph.h b/libs/gltfio/src/DependencyGraph.h index 9b2928f607..8320b9ee4e 100644 --- a/libs/gltfio/src/DependencyGraph.h +++ b/libs/gltfio/src/DependencyGraph.h @@ -118,8 +118,8 @@ private: TextureNode* getStatus(filament::Texture* texture); // The following maps contain the directed edges in the graph. - tsl::robin_map mEntityToMaterial; - tsl::robin_map> mMaterialToEntity; + tsl::robin_map mEntityToMaterial; + tsl::robin_map> mMaterialToEntity; tsl::robin_map mMaterialToTexture; tsl::robin_map> mTextureToMaterial; diff --git a/libs/gltfio/src/FFilamentInstance.h b/libs/gltfio/src/FFilamentInstance.h index 52cfdd77fa..8423147e1c 100644 --- a/libs/gltfio/src/FFilamentInstance.h +++ b/libs/gltfio/src/FFilamentInstance.h @@ -55,7 +55,7 @@ struct Skin { // The set of all nodes that are influenced by this skin. // This is initially gleaned from the glTF file using the "skin" attribute of each node. - tsl::robin_set targets; + tsl::robin_set targets; }; struct VariantMapping { diff --git a/libs/gltfio/src/GltfEnums.h b/libs/gltfio/src/GltfEnums.h index 39e71b7a0c..245240cfd2 100644 --- a/libs/gltfio/src/GltfEnums.h +++ b/libs/gltfio/src/GltfEnums.h @@ -263,11 +263,4 @@ inline bool getElementType(cgltf_type type, cgltf_component_type ctype, return false; } -inline bool requiresConversion(cgltf_type type, cgltf_component_type ctype) { - filament::VertexBuffer::AttributeType permitted; - filament::VertexBuffer::AttributeType actual; - bool supported = getElementType(type, ctype, &permitted, &actual); - return supported && permitted != actual; -} - #endif // GLTFIO_GLTFENUMS_H diff --git a/libs/gltfio/src/ResourceLoader.cpp b/libs/gltfio/src/ResourceLoader.cpp index da19320a0e..c43f69ff87 100644 --- a/libs/gltfio/src/ResourceLoader.cpp +++ b/libs/gltfio/src/ResourceLoader.cpp @@ -176,8 +176,9 @@ static ComponentType getComponentType(const cgltf_accessor* accessor) { case cgltf_component_type_r_8u: return ComponentType::UBYTE; case cgltf_component_type_r_16: return ComponentType::SHORT; case cgltf_component_type_r_16u: return ComponentType::USHORT; + case cgltf_component_type_r_32f: return ComponentType::FLOAT; + case cgltf_component_type_r_32u: default: - // This should be unreachable because other types do not require conversion. assert_invariant(false); return {}; } @@ -197,6 +198,34 @@ static void convertToFloats(float* dest, const cgltf_accessor* accessor) { transcode(dest, source, accessor->count); } +static bool requiresConversion(cgltf_type type, cgltf_component_type ctype) { + filament::VertexBuffer::AttributeType permitted; + filament::VertexBuffer::AttributeType actual; + bool supported = getElementType(type, ctype, &permitted, &actual); + return supported && permitted != actual; +} + +static bool requiresPacking(const cgltf_accessor* accessor) { + if (requiresConversion(accessor->type, accessor->component_type)) { + return true; + } + const size_t dim = cgltf_num_components(accessor->type); + switch (accessor->component_type) { + case cgltf_component_type_r_8: + case cgltf_component_type_r_8u: + return accessor->stride != dim; + case cgltf_component_type_r_16: + case cgltf_component_type_r_16u: + return accessor->stride != dim * 2; + case cgltf_component_type_r_32u: + case cgltf_component_type_r_32f: + return accessor->stride != dim * 4; + default: + assert_invariant(false); + return true; + } +} + static void decodeDracoMeshes(FFilamentAsset* asset) { DracoCache* dracoCache = &asset->mSourceAsset->dracoCache; @@ -492,7 +521,27 @@ bool ResourceLoader::loadResources(FFilamentAsset* asset, bool async) { slot.indexBuffer->setBuffer(engine, std::move(bd)); continue; } + + // If the buffer slot does not have an associated VertexBuffer or IndexBuffer, then this + // must be a morph target. assert(slot.morphTargetBuffer); + + if (requiresPacking(accessor)) { + const size_t dim = cgltf_num_components(accessor->type); + const size_t floatsSize = accessor->count * sizeof(float) * dim; + float* floatsData = (float*) malloc(floatsSize); + convertToFloats(floatsData, accessor); + if (accessor->type == cgltf_type_vec3) { + slot.morphTargetBuffer->setPositionsAt(engine, slot.bufferIndex, + (const float3*) floatsData, slot.morphTargetBuffer->getVertexCount()); + } else { + slot.morphTargetBuffer->setPositionsAt(engine, slot.bufferIndex, + (const float4*) data, slot.morphTargetBuffer->getVertexCount()); + } + free(floatsData); + continue; + } + if (accessor->type == cgltf_type_vec3) { slot.morphTargetBuffer->setPositionsAt(engine, slot.bufferIndex, (const float3*) data, slot.morphTargetBuffer->getVertexCount()); diff --git a/libs/gltfio/src/UbershaderLoader.cpp b/libs/gltfio/src/UbershaderLoader.cpp index de08336813..12e8b88487 100644 --- a/libs/gltfio/src/UbershaderLoader.cpp +++ b/libs/gltfio/src/UbershaderLoader.cpp @@ -24,17 +24,20 @@ #include -#if GLTFIO_LITE -#include "gltfresources_lite.h" -#else +#include + #include "gltfresources.h" -#endif using namespace filament; using namespace filament::math; +using namespace filament::uberz; using namespace gltfio; using namespace utils; +#if !defined(NDEBUG) +utils::io::ostream& operator<<(utils::io::ostream& out, const filament::uberz::ArchiveRequirements& reqs); +#endif + namespace { using CullingMode = MaterialInstance::CullingMode; @@ -64,35 +67,13 @@ public: Material* getMaterial(const MaterialKey& config) const; - enum ShadingMode { - UNLIT = 0, - LIT = 1, - SPECULAR_GLOSSINESS = 2, - }; - - mutable Material* mMaterials[12] = {}; + mutable ArchiveCache mMaterials; Texture* mDummyTexture = nullptr; Engine* mEngine; }; -#if GLTFIO_LITE - -#define CREATE_MATERIAL(name) Material::Builder() \ - .package(GLTFRESOURCES_LITE_ ## name ## _DATA, GLTFRESOURCES_LITE_ ## name ## _SIZE) \ - .build(*mEngine); - -#else - -#define CREATE_MATERIAL(name) Material::Builder() \ - .package(GLTFRESOURCES_ ## name ## _DATA, GLTFRESOURCES_ ## name ## _SIZE) \ - .build(*mEngine); - -#endif - -#define MATINDEX(shading, alpha, sheen, transmit, volume) (volume ? 11 : (transmit ? 10 : (sheen ? 9 : (int(shading) + 3 * int(alpha))))) - -UbershaderLoader::UbershaderLoader(Engine* engine) : mEngine(engine) { +UbershaderLoader::UbershaderLoader(Engine* engine) : mEngine(engine), mMaterials(*engine) { unsigned char texels[4] = {}; mDummyTexture = Texture::Builder() .width(1).height(1) @@ -101,65 +82,63 @@ UbershaderLoader::UbershaderLoader(Engine* engine) : mEngine(engine) { Texture::PixelBufferDescriptor pbd(texels, sizeof(texels), Texture::Format::RGBA, Texture::Type::UBYTE); mDummyTexture->setImage(*mEngine, 0, std::move(pbd)); + mMaterials.load((void*) GLTFRESOURCES_FULL_DATA, GLTFRESOURCES_FULL_SIZE); } size_t UbershaderLoader::getMaterialsCount() const noexcept { - return sizeof(mMaterials) / sizeof(mMaterials[0]); + return mMaterials.getMaterialsCount(); } const Material* const* UbershaderLoader::getMaterials() const noexcept { - return &mMaterials[0]; + return mMaterials.getMaterials(); } void UbershaderLoader::destroyMaterials() { - for (auto& material : mMaterials) { - mEngine->destroy(material); - material = nullptr; - } + mMaterials.destroyMaterials(); mEngine->destroy(mDummyTexture); } Material* UbershaderLoader::getMaterial(const MaterialKey& config) const { - const ShadingMode shading = config.unlit ? UNLIT : - (config.useSpecularGlossiness ? SPECULAR_GLOSSINESS : LIT); - const int matindex = MATINDEX(shading, config.alphaMode, config.hasSheen, config.hasTransmission, config.hasVolume); - if (mMaterials[matindex] != nullptr) { - return mMaterials[matindex]; - } - switch (matindex) { - #if !GLTFIO_LITE || defined(GLTFRESOURCES_LITE_LIT_OPAQUE_DATA) - case MATINDEX(LIT, AlphaMode::OPAQUE, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(LIT_OPAQUE); break; - #endif + const Shading shading = config.unlit ? Shading::UNLIT : + (config.useSpecularGlossiness ? Shading::SPECULAR_GLOSSINESS : Shading::LIT); - #if !GLTFIO_LITE || defined(GLTFRESOURCES_LITE_LIT_FADE_DATA) - case MATINDEX(LIT, AlphaMode::BLEND, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(LIT_FADE); break; - #endif + BlendingMode blending; + switch (config.alphaMode) { + case AlphaMode::OPAQUE: blending = BlendingMode::OPAQUE; break; + case AlphaMode::MASK: blending = BlendingMode::MASKED; break; + case AlphaMode::BLEND: blending = BlendingMode::FADE; break; + } - #if !GLTFIO_LITE - case MATINDEX(LIT, AlphaMode::MASK, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(LIT_MASKED); break; - case MATINDEX(UNLIT, AlphaMode::OPAQUE, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(UNLIT_OPAQUE); break; - case MATINDEX(UNLIT, AlphaMode::MASK, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(UNLIT_MASKED); break; - case MATINDEX(UNLIT, AlphaMode::BLEND, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(UNLIT_FADE); break; - case MATINDEX(SPECULAR_GLOSSINESS, AlphaMode::OPAQUE, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(SPECULARGLOSSINESS_OPAQUE); break; - case MATINDEX(SPECULAR_GLOSSINESS, AlphaMode::MASK, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(SPECULARGLOSSINESS_MASKED); break; - case MATINDEX(SPECULAR_GLOSSINESS, AlphaMode::BLEND, false, false, false): mMaterials[matindex] = CREATE_MATERIAL(SPECULARGLOSSINESS_FADE); break; - case MATINDEX(0, 0, false, true, false): mMaterials[matindex] = CREATE_MATERIAL(LIT_TRANSMISSION); break; - case MATINDEX(0, 0, true, false, false): mMaterials[matindex] = CREATE_MATERIAL(LIT_SHEEN); break; - case MATINDEX(0, 0, false, false, true): mMaterials[matindex] = CREATE_MATERIAL(LIT_VOLUME); break; - #endif + ArchiveRequirements requirements = { shading, blending }; + requirements.features["Sheen"] = config.hasSheen; + requirements.features["Transmission"] = config.hasTransmission; + requirements.features["Volume"] = config.hasVolume; + requirements.features["Ior"] = config.hasIOR; + requirements.features["VertexColors"] = config.hasVertexColors; + requirements.features["BaseColorTexture"] = config.hasBaseColorTexture; + requirements.features["NormalTexture"] = config.hasNormalTexture; + requirements.features["OcclusionTexture"] = config.hasOcclusionTexture; + requirements.features["EmissiveTexture"] = config.hasEmissiveTexture; + requirements.features["MetallicRoughnessTexture"] = config.hasMetallicRoughnessTexture; + requirements.features["ClearCoatTexture"] = config.hasClearCoatTexture; + requirements.features["ClearCoatRoughnessTexture"] = config.hasClearCoatRoughnessTexture; + requirements.features["ClearCoatNormalTexture"] = config.hasClearCoatNormalTexture; + requirements.features["ClearCoat"] = config.hasClearCoat; + requirements.features["TextureTransforms"] = config.hasTextureTransforms; + requirements.features["TransmissionTexture"] = config.hasTransmissionTexture; + requirements.features["SheenColorTexture"] = config.hasSheenColorTexture; + requirements.features["SheenRoughnessTexture"] = config.hasSheenRoughnessTexture; + requirements.features["VolumeThicknessTexture"] = config.hasVolumeThicknessTexture; + + if (Material* mat = mMaterials.getMaterial(requirements); mat != nullptr) { + return mat; } - if (mMaterials[matindex] == nullptr) { - slog.w << "Unsupported glTF material configuration; falling back to LIT_OPAQUE." << io::endl; - MaterialKey litOpaque = config; - litOpaque.alphaMode = AlphaMode::OPAQUE; - litOpaque.hasTransmission = false; - litOpaque.hasVolume = false; - litOpaque.hasSheen = false; - litOpaque.useSpecularGlossiness = false; - litOpaque.unlit = false; - return getMaterial(litOpaque); - } - return mMaterials[matindex]; + +#ifndef NDEBUG + slog.w << "Failed to find material with features:\n" << requirements << io::endl; +#endif + + return nullptr; } MaterialInstance* UbershaderLoader::createMaterialInstance(MaterialKey* config, UvMap* uvmap, @@ -197,6 +176,14 @@ MaterialInstance* UbershaderLoader::createMaterialInstance(MaterialKey* config, return hasTexture ? int(uvmap->at(srcIndex)) - 1 : -1; }; Material* material = getMaterial(*config); + + if (material == nullptr) { +#ifndef NDEBUG + slog.w << "Using fallback material for " << label << "." << io::endl; +#endif + material = mMaterials.getDefaultMaterial(); + } + MaterialInstance* mi = material->createInstance(label); mi->setParameter("baseColorIndex", getUvIndex(config->baseColorUV, config->hasBaseColorTexture)); @@ -212,8 +199,6 @@ MaterialInstance* UbershaderLoader::createMaterialInstance(MaterialKey* config, MaterialInstance::TransparencyMode::TWO_PASSES_TWO_SIDES : MaterialInstance::TransparencyMode::DEFAULT); - #if !GLTFIO_LITE - // Initially, assume that the clear coat texture can be honored. This is changed to false when // running into a sampler count limitation. TODO: check if these constraints can now be relaxed. bool clearCoatNeedsTexture = true; @@ -262,17 +247,6 @@ MaterialInstance* UbershaderLoader::createMaterialInstance(MaterialKey* config, getUvIndex(config->transmissionUV, config->hasTransmissionTexture)); } } - #else - - // In the GLTFIO_LITE configuration we do not support UV matrices, clear coat, sheen, specular - // glossiness, or transmission. For more details, see `gltflite.mat.in`. To configure a custom - // set of features, create your own MaterialProvider class, perhaps using UbershaderLoader as a - // starting point. - const bool clearCoatNeedsTexture = false; - - const bool volumeThicknessNeedsTexture = false; - - #endif TextureSampler sampler; mi->setParameter("normalMap", mDummyTexture, sampler); @@ -317,3 +291,41 @@ MaterialProvider* createUbershaderLoader(filament::Engine* engine) { } } // namespace gltfio + + +#if !defined(NDEBUG) + +inline +const char* toString(Shading shadingModel) noexcept { + switch (shadingModel) { + case Shading::UNLIT: return "unlit"; + case Shading::LIT: return "lit"; + case Shading::SUBSURFACE: return "subsurface"; + case Shading::CLOTH: return "cloth"; + case Shading::SPECULAR_GLOSSINESS: return "specularGlossiness"; + } +} + +inline +const char* toString(BlendingMode blendingMode) noexcept { + switch (blendingMode) { + case BlendingMode::OPAQUE: return "opaque"; + case BlendingMode::TRANSPARENT: return "transparent"; + case BlendingMode::ADD: return "add"; + case BlendingMode::MASKED: return "masked"; + case BlendingMode::FADE: return "fade"; + case BlendingMode::MULTIPLY: return "multiply"; + case BlendingMode::SCREEN: return "screen"; + } +} + +io::ostream& operator<<(io::ostream& out, const ArchiveRequirements& reqs) { + out << " ShadingModel = " << toString(reqs.shadingModel) << '\n' + << " BlendingMode = " << toString(reqs.blendingMode) << '\n'; + for (const auto& pair : reqs.features) { + out << " " << pair.first.c_str() << " = " << (pair.second ? "true" : "false") << '\n'; + } + return out; +} + +#endif diff --git a/libs/image/src/ImageSampler.cpp b/libs/image/src/ImageSampler.cpp index 98073c4894..d922e505bc 100644 --- a/libs/image/src/ImageSampler.cpp +++ b/libs/image/src/ImageSampler.cpp @@ -364,7 +364,7 @@ uint32_t getMipmapCount(const LinearImage& source) { Filter filterFromString(const char* rawname) { using namespace utils; using std::unordered_map; - static const unordered_map map = { + static const unordered_map map = { { "BOX", Filter::BOX}, { "NEAREST", Filter::NEAREST}, { "HERMITE", Filter::HERMITE}, diff --git a/libs/uberz/CMakeLists.txt b/libs/uberz/CMakeLists.txt new file mode 100644 index 0000000000..19f84c275c --- /dev/null +++ b/libs/uberz/CMakeLists.txt @@ -0,0 +1,48 @@ +cmake_minimum_required(VERSION 3.19) +project(uberzlib) + +set(TARGET uberzlib) +set(PUBLIC_HDR_DIR include) + +# ================================================================================================== +# Sources and headers +# ================================================================================================== +set(PUBLIC_HDRS + include/uberz/ArchiveCache.h + include/uberz/ArchiveEnums.h + include/uberz/ReadableArchive.h + include/uberz/WritableArchive.h +) + +set(SRCS + src/ArchiveCache.cpp + src/ReadableArchive.cpp + src/WritableArchive.cpp +) + +# ================================================================================================== +# Include and target definitions +# ================================================================================================== +include_directories(${PUBLIC_HDR_DIR}) + +add_library(${TARGET} STATIC ${PUBLIC_HDRS} ${SRCS}) + +target_link_libraries(${TARGET} PUBLIC math utils filabridge filament zstd) + +target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) + +# ================================================================================================== +# Compiler flags +# ================================================================================================== +if (MSVC) + target_compile_options(${TARGET} PRIVATE $<$:/fp:fast>) +else() + target_compile_options(${TARGET} PRIVATE $<$:-ffast-math>) + target_compile_options(${TARGET} PRIVATE -Wno-deprecated-register) +endif() + +# ================================================================================================== +# Installation +# ================================================================================================== +install(TARGETS ${TARGET} ARCHIVE DESTINATION lib/${DIST_DIR}) +install(DIRECTORY ${PUBLIC_HDR_DIR}/uberz DESTINATION include) diff --git a/libs/uberz/README.md b/libs/uberz/README.md new file mode 100644 index 0000000000..9862e3417f --- /dev/null +++ b/libs/uberz/README.md @@ -0,0 +1,87 @@ +- [Ubershader Archive Files](#ubershader-archive-files) +- [Ubershader Spec Files](#ubershader-spec-files) + +# Ubershader Archive Files + +An ubershader archive provides a way to bundle up a set of `filamat` files along with some metadata +that conveys which glTF features each material can handle. It is a file that has been compressed +with `zstd` and has an `.uberz` file extension. In uncompressed form, it has the following layout +(little endian is assumed). + +``` +[u32] magic identifier: UBER +[u32] simple (unpartitioned) version number for the archive format +[u64] number of specs +[u64] byte offset to SPECS +SPECS: +foreach spec { + [u8] shading model + [u8] blending model + [u16] number of flags + [u32] size in bytes of the filamat blob + [u64] byte offset to FLAGLIST for this spec + [u64] byte offset to FILAMAT for this spec +} +foreach spec { + FLAGLIST: + foreach flag { + [u64] byte offset to FLAGNAME for this spec/flag pair + [u64] flag value: 0 = unsupported, 1 = optional, or 2 = required + } +} +foreach spec { + foreach flag { + FLAGNAME: + [u8...] flag name, including null terminator + } +} +foreach spec { + FILAMAT: + [u8...] filamat blob +} +``` + +In the above specification, each "offset" is a number of bytes between the top of the file to the +given label. These offsets are 64 bits so that they can be replaced with pointers in a C struct, +which allows the file to be consumed without any parsing. On 32-bit architectures, this still works +because we can simply ignore the unused padding after every pointer. + +# Ubershader Spec Files + +An ubershader spec file is a simple text file with a `.spec` extension. It contains a list of +key-value pairs conforming to the following grammar. Each key-value pair is either a *feature flag* +or a *fundamental aspect*. + +- Each feature flag can be **unsupported**, **required**, or **optional**. +- The fundamental aspect of the material cannot be changed, such as the blend mode. + +```eBNF +spec = { [ comment | key_value_pair ] , "\n" } ; +comment = "#" , { any } ; +key_value_pair = ( fundamental_aspect | feature_flag ) ; +fundamental_aspect = ( blending | shading ) ; +feature_flag = identifier , equals , ("unsupported" | "required" | "optional") ; +blending = "BlendingMode" , equals , + ( "opaque" | "transparent" | "fade" | "add" | "masked" | "multiply" | "screen" ) ; +shading = "ShadingModel" , equals , + ( "lit" | "subsurface" | "cloth" | "unlit" | "specularGlossiness") ; +equals = [ whitespace ] , "=" , [ whitespace ] ; +any = ? any character other than newline ? ; +whitespace = ? sequence of tabs and spaces ? ; +identifier = ? sequence of alphanumeric characters ? ; +``` + +If a fundamental aspect is missing from the spec, then the loader will assume that the spec can +handle all possible values for that aspect. For example, we may wish to override the glTF +blending mode in certain ubershader materials (e.g. materials that support `KHR_materials_volume`). +These materials should simply omit the `BlendingMode` line from the spec. + +If any feature flag is missing from the spec, it implicitly has the value of `unsupported`. For an +up-to-date list of recognized feature flags, look at the source for `UbershaderLoader::getMaterial`. + +If a particular feature flag is set to `required` for a particular material, then the glTF loader +will bind that material to a given glTF mesh only if that feature is enabled in the mesh. + +Usually, features are either `unsupported` or `optional`. For example, if the ubershader user can +set `normalIndex` in the material to `-1` to signal that they do not have a normal map, then normal +mapping should be specified as an `optional` feature of the ubershader. diff --git a/libs/uberz/include/uberz/ArchiveCache.h b/libs/uberz/include/uberz/ArchiveCache.h new file mode 100644 index 0000000000..2c62ceef9c --- /dev/null +++ b/libs/uberz/include/uberz/ArchiveCache.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UBERZ_ARCHIVE_CACHE_H +#define UBERZ_ARCHIVE_CACHE_H + +#include +#include + +#include + +#include + +#include +#include + +namespace filament::uberz { + + struct ArchiveRequirements; + + // Stores a set of Filament materials and knows how to choose a suitable material when given a + // set of requirements. Used by gltfio; users do not need to access this class directly. + class ArchiveCache { + public: + ArchiveCache(filament::Engine& engine) : mEngine(engine) {} + ~ArchiveCache(); + + void load(const void* archiveData, uint64_t archiveByteCount); + filament::Material* getMaterial(const ArchiveRequirements& requirements); + filament::Material* getDefaultMaterial(); + const filament::Material* const* getMaterials() const noexcept { return mMaterials.data(); } + size_t getMaterialsCount() const noexcept { return mMaterials.size(); } + void destroyMaterials(); + + private: + filament::Engine& mEngine; + utils::FixedCapacityVector mMaterials; + struct ReadableArchive* mArchive = nullptr; + }; + + struct ArchiveRequirements { + filament::Shading shadingModel; + filament::BlendingMode blendingMode; + tsl::robin_map features; + }; + +} // namespace filament::uberz + +#endif // UBERZ_ARCHIVE_CACHE_H diff --git a/libs/uberz/include/uberz/ArchiveEnums.h b/libs/uberz/include/uberz/ArchiveEnums.h new file mode 100644 index 0000000000..b2741be939 --- /dev/null +++ b/libs/uberz/include/uberz/ArchiveEnums.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UBERZ_ARCHIVE_ENUMS_H +#define UBERZ_ARCHIVE_ENUMS_H + +#include + +namespace filament::uberz { + + enum class ArchiveFeature : uint64_t { + UNSUPPORTED, + OPTIONAL, + REQUIRED, + }; + +} // namespace filament::uberz + +#endif // UBERZ_ARCHIVE_ENUMS_H diff --git a/libs/uberz/include/uberz/ReadableArchive.h b/libs/uberz/include/uberz/ReadableArchive.h new file mode 100644 index 0000000000..bae726b530 --- /dev/null +++ b/libs/uberz/include/uberz/ReadableArchive.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UBERZ_READABLE_ARCHIVE_H +#define UBERZ_READABLE_ARCHIVE_H + +#include + +#include + +#include + +namespace filament::uberz { + +// ArchiveSpec is a parse-free binary format. The client simply casts a word-aligned content blob +// into a ReadableArchive struct pointer, then calls the following function to convert all the +// offset fields into pointers. +void convertOffsetsToPointers(struct ReadableArchive* archive); + +#pragma clang diagnostic push +#pragma clang diagnostic warning "-Wpadded" + +// Precompiled set of materials bundled with a list of features flags that each material supports. +// This is the readable counterpart to WriteableArchive. +// Used by gltfio; users do not need to access this class directly. +struct ReadableArchive { + uint32_t magic; + uint32_t version; + uint64_t specsCount; + union { + struct ArchiveSpec* specs; + uint64_t specsOffset; + }; +}; + +static constexpr filament::Shading INVALID_SHADING_MODEL = (filament::Shading) 0xff; +static constexpr filament::BlendingMode INVALID_BLENDING = (filament::BlendingMode) 0xff; + +struct ArchiveSpec { + filament::Shading shadingModel; + filament::BlendingMode blendingMode; + uint16_t flagsCount; + uint32_t packageByteCount; + union { + struct ArchiveFlag* flags; + uint64_t flagsOffset; + }; + union { + uint8_t* package; + uint64_t packageOffset; + }; +}; + +struct ArchiveFlag { + union { + const char* name; + uint64_t nameOffset; + }; + ArchiveFeature value; +}; + +#pragma clang diagnostic pop + +} // namespace filament::uberz + +#endif // UBERZ_READABLE_ARCHIVE_H diff --git a/libs/uberz/include/uberz/WritableArchive.h b/libs/uberz/include/uberz/WritableArchive.h new file mode 100644 index 0000000000..c25c8bcab0 --- /dev/null +++ b/libs/uberz/include/uberz/WritableArchive.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UBERZ_WRITABLE_ARCHIVE_H +#define UBERZ_WRITABLE_ARCHIVE_H + +#include + +#include +#include +#include + +#include + +#include + +namespace filament::uberz { + +// Precompiled set of materials bundled with a list of features flags that each material supports. +// This is the writeable counterpart to ReadableArchive. +// Used by gltfio; users do not need to access this class directly. +class WritableArchive { +public: + WritableArchive(size_t materialCount) : mMaterials(materialCount) {} + void addMaterial(const char* name, const uint8_t* package, size_t packageSize); + void addSpecLine(std::string_view line); + utils::FixedCapacityVector serialize() const; + +private: + size_t mLineNumber = 1; + ssize_t mMaterialIndex = -1; + + struct Material { + utils::CString name; + utils::FixedCapacityVector package; + filament::Shading shadingModel; + filament::BlendingMode blendingMode; + tsl::robin_map flags; + }; + + utils::FixedCapacityVector mMaterials; +}; + +} // namespace filament::uberz + +#endif // UBERZ_WRITABLE_ARCHIVE_H diff --git a/libs/uberz/src/ArchiveCache.cpp b/libs/uberz/src/ArchiveCache.cpp new file mode 100644 index 0000000000..f9bfe863a8 --- /dev/null +++ b/libs/uberz/src/ArchiveCache.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include + +using namespace utils; + +namespace filament::uberz { + + +// Set this to a certain spec index to find out why it was deemed unsuitable. +// To find the spec index of interest, try invoking uberz with the verbose flag. +constexpr static int DEBUG_SPEC_INDEX = -1; + +constexpr static void debugSuitability(int index, const char* msg) { + if constexpr(DEBUG_SPEC_INDEX > 0) { + if (index == DEBUG_SPEC_INDEX) { + slog.d << "Spec " << DEBUG_SPEC_INDEX + << " is unsuitable due to " << msg << io::endl; + } + } +} + +static bool strIsEqual(const CString& a, const char* b) { + return strncmp(a.c_str(), b, a.size()) == 0; +} + +void ArchiveCache::load(const void* archiveData, uint64_t archiveByteCount) { + assert_invariant(mArchive == nullptr && "Do not call load() twice"); + const uint64_t decompSize = ZSTD_getFrameContentSize(archiveData, archiveByteCount); + if (decompSize == ZSTD_CONTENTSIZE_UNKNOWN || decompSize == ZSTD_CONTENTSIZE_ERROR) { + PANIC_POSTCONDITION("Decompression error."); + } + uint64_t* basePointer = (uint64_t*) utils::aligned_alloc(decompSize, 8); + ZSTD_decompress(basePointer, decompSize, archiveData, archiveByteCount); + mArchive = (ReadableArchive*) basePointer; + convertOffsetsToPointers(mArchive); + mMaterials = FixedCapacityVector(mArchive->specsCount, nullptr); +} + +// This loops though all ubershaders and returns the first one that meets the given requirements. +Material* ArchiveCache::getMaterial(const ArchiveRequirements& reqs) { + assert_invariant(mArchive && "Please call load() before requesting any materials."); + + for (uint64_t i = 0; i < mArchive->specsCount; ++i) { + const ArchiveSpec& spec = mArchive->specs[i]; + if (spec.blendingMode != INVALID_BLENDING && spec.blendingMode != reqs.blendingMode) { + debugSuitability(i, "blend mode mismatch."); + continue; + } + if (spec.shadingModel != INVALID_SHADING_MODEL && spec.shadingModel != reqs.shadingModel) { + debugSuitability(i, "material model."); + continue; + } + bool specIsSuitable = true; + + // For each feature required by the mesh, this ubershader is suitable only if it includes a + // feature flag for it and the feature flag is either OPTIONAL or REQUIRED. + for (const auto& req : reqs.features) { + const CString& meshRequirement = req.first; + if (req.second == false) { + continue; + } + bool found = false; + for (uint64_t j = 0; j < spec.flagsCount && !found; ++j) { + const ArchiveFlag& flag = spec.flags[j]; + if (strIsEqual(meshRequirement, flag.name)) { + if (flag.value != ArchiveFeature::UNSUPPORTED) { + found = true; + } + break; + } + } + if (!found) { + debugSuitability(i, meshRequirement.c_str()); + specIsSuitable = false; + break; + } + } + + // If this ubershader requires a certain feature to be enabled in the glTF, but the glTF + // mesh doesn't have it, then this ubershader is not suitable. This occurs very rarely, so + // it intentionally comes after the other suitability check. + for (uint64_t j = 0; j < spec.flagsCount && specIsSuitable; ++j) { + ArchiveFlag& flag = spec.flags[j]; + if (UTILS_UNLIKELY(flag.value == ArchiveFeature::REQUIRED)) { + // This allocates a new CString just to make a robin_map lookup, but this is rare + // because almost none of our feature flags are REQUIRED. + auto iter = reqs.features.find(CString(flag.name)); + if (iter == reqs.features.end() || iter.value() == false) { + debugSuitability(i, flag.name); + specIsSuitable = false; + } + } + } + + if (specIsSuitable) { + if (mMaterials[i] == nullptr) { + mMaterials[i] = Material::Builder() + .package(spec.package, spec.packageByteCount) + .build(mEngine); + } + return mMaterials[i]; + } + } + return nullptr; +} + +Material* ArchiveCache::getDefaultMaterial() { + assert_invariant(mArchive && "Please call load() before requesting any materials."); + assert_invariant(!mMaterials.empty() && "Archive must have at least one material."); + if (mMaterials[0] == nullptr) { + mMaterials[0] = Material::Builder() + .package(mArchive->specs[0].package, mArchive->specs[0].packageByteCount) + .build(mEngine); + } + return mMaterials[0]; +} + +void ArchiveCache::destroyMaterials() { + for (auto mat : mMaterials) mEngine.destroy(mat); + mMaterials.clear(); +} + +ArchiveCache::~ArchiveCache() { + assert_invariant(mMaterials.size() == 0 && + "Please call destroyMaterials explicitly to ensure correct destruction order"); + free(mArchive); +} + +} // namespace filament::uberz diff --git a/libs/uberz/src/ReadableArchive.cpp b/libs/uberz/src/ReadableArchive.cpp new file mode 100644 index 0000000000..30ce30330a --- /dev/null +++ b/libs/uberz/src/ReadableArchive.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +using namespace filament; +using namespace utils; + +namespace filament::uberz { + +static_assert(sizeof(ReadableArchive) == 4 + 4 + 8 + 8); +static_assert(sizeof(ArchiveSpec) == 1 + 1 + 2 + 4 + 8 + 8); +static_assert(sizeof(ArchiveFlag) == 8 + 8); + +void convertOffsetsToPointers(ReadableArchive* archive) { + constexpr size_t wordSize = sizeof(uint64_t); + assert_invariant(archive->specsOffset % wordSize == 0); + uint64_t* basePointer = (uint64_t*) archive; + archive->specs = (ArchiveSpec*) (basePointer + archive->specsOffset / wordSize); + for (uint64_t i = 0; i < archive->specsCount; ++i) { + ArchiveSpec& spec = archive->specs[i]; + assert_invariant(spec.flagsOffset % wordSize == 0); + spec.flags = (ArchiveFlag*) (basePointer + (spec.flagsOffset / wordSize)); + spec.package = ((uint8_t*) basePointer) + spec.packageOffset; + for (uint64_t j = 0; j < spec.flagsCount; ++j) { + ArchiveFlag& flag = spec.flags[j]; + flag.name = ((const char*) basePointer) + flag.nameOffset; + } + } +} + +} // namespace filament::uberz diff --git a/libs/uberz/src/WritableArchive.cpp b/libs/uberz/src/WritableArchive.cpp new file mode 100644 index 0000000000..67fd995e94 --- /dev/null +++ b/libs/uberz/src/WritableArchive.cpp @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include + +#include + +using namespace utils; +using namespace std::literals; + +using std::string_view; + +namespace filament::uberz { + +static bool isAlphaNumeric(char c) { + return (c >= '0' && c <= '9') || c == '_' + || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool isWhitespace(char c) { + return c == ' ' || c == '\t'; +} + +// Returns 0 if stringToScan does not start with prefix, otherwise returns length of prefix. +static size_t readPrefix(string_view prefix, string_view stringToScan) { + const size_t prefixSize = prefix.size(); + if (UTILS_UNLIKELY(stringToScan.size() < prefixSize)) { + return 0; + } + return prefix == string_view { stringToScan.data(), prefixSize } ? prefixSize : 0; +} + +static string_view readArchiveFeature(string_view cursor, ArchiveFeature* feature) { + size_t sz; + if (sz = readPrefix("unsupported"sv, cursor); sz > 0) { + *feature = ArchiveFeature::UNSUPPORTED; + } else if (sz = readPrefix("required"sv, cursor); sz > 0) { + *feature = ArchiveFeature::REQUIRED; + } else if (sz = readPrefix("optional"sv, cursor); sz > 0) { + *feature = ArchiveFeature::OPTIONAL; + } + return { cursor.data(), sz }; +} + +static string_view readBlendingMode(string_view cursor, BlendingMode* blending) { + size_t sz; + if (sz = readPrefix("opaque"sv, cursor); sz > 0) { + *blending = BlendingMode::OPAQUE; + } else if (sz = readPrefix("transparent"sv, cursor); sz > 0) { + *blending = BlendingMode::TRANSPARENT; + } else if (sz = readPrefix("add"sv, cursor); sz > 0) { + *blending = BlendingMode::ADD; + } else if (sz = readPrefix("masked"sv, cursor); sz > 0) { + *blending = BlendingMode::MASKED; + } else if (sz = readPrefix("fade"sv, cursor); sz > 0) { + *blending = BlendingMode::FADE; + } else if (sz = readPrefix("multiply"sv, cursor); sz > 0) { + *blending = BlendingMode::MULTIPLY; + } else if (sz = readPrefix("screen"sv, cursor); sz > 0) { + *blending = BlendingMode::SCREEN; + } + return { cursor.data(), sz }; +} + +static string_view readShadingModel(string_view cursor, Shading* shading) { + size_t sz; + if (sz = readPrefix("unlit"sv, cursor); sz > 0) { + *shading = Shading::UNLIT; + } else if (sz = readPrefix("lit"sv, cursor); sz > 0) { + *shading = Shading::LIT; + } else if (sz = readPrefix("subsurface"sv, cursor); sz > 0) { + *shading = Shading::SUBSURFACE; + } else if (sz = readPrefix("cloth"sv, cursor); sz > 0) { + *shading = Shading::CLOTH; + } else if (sz = readPrefix("specularGlossiness"sv, cursor); sz > 0) { + *shading = Shading::SPECULAR_GLOSSINESS; + } + return { cursor.data(), sz }; +} + +static string_view readSymbol(string_view cursor, char symbol) { + return { cursor.data(), cursor[0] == symbol ? 1ul : 0ul }; +} + +static string_view readIdentifier(string_view cursor) { + size_t i = 0; + while (isAlphaNumeric(cursor[i])) i++; + return { cursor.data(), i }; +} + +static string_view readWhitespace(string_view cursor) { + size_t i = 0; + while (isWhitespace(cursor[i])) i++; + return { cursor.data(), i }; +} + +void WritableArchive::addMaterial(const char* name, const uint8_t* package, size_t packageSize) { + Material& material = mMaterials[++mMaterialIndex]; + + material = { + CString(name), + FixedCapacityVector(packageSize) + }; + memcpy(material.package.data(), package, packageSize); + + // We use invalid values to denote "not set". e.g. if the spec file + // does not set the blend mode, then it can be used for any blend mode. + material.blendingMode = INVALID_BLENDING; + material.shadingModel = INVALID_SHADING_MODEL; + + mLineNumber = 1; +} + +void WritableArchive::addSpecLine(string_view line) { + assert_invariant(mMaterialIndex > -1); + Material& material = mMaterials[mMaterialIndex]; + + string_view cursor = line; + string_view token; + + // Don't try to recover, just emit the error message and quit. + auto emitSyntaxError = [&](const char* msg) { + const int column = 1 + line.size() - cursor.size(); + PANIC_POSTCONDITION("%s.spec(%d,%d): %s", material.name.c_str(), mLineNumber, column, msg); + }; + + // Advance the cursor forward each time a token is consumed. + auto advanceCursorByToken = [&]() { + cursor = { cursor.data() + token.size(), cursor.size() - token.size() }; + }; + + // Sometimes we need to advance the cursor forward by a fixed amount. + auto advanceCursorByCount = [&](size_t count) { + cursor = { cursor.data() + count, cursor.size() - count }; + }; + + auto parseFeatureFlag = [&]() { + if (token = readIdentifier(cursor); token.empty()) { + emitSyntaxError("expected identifier"); + } + advanceCursorByToken(); + CString id(token.data(), token.size()); + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readSymbol(cursor, '='); token.empty()) { + emitSyntaxError("expected equal sign"); + } + advanceCursorByToken(); + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readArchiveFeature(cursor, &material.flags[id]); token.empty()) { + emitSyntaxError("expected unsupported / optional / required"); + } + advanceCursorByToken(); + }; + + auto parseBlendingMode = [&]() { + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readSymbol(cursor, '='); token.empty()) { + emitSyntaxError("expected equal sign"); + } + advanceCursorByToken(); + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readBlendingMode(cursor, &material.blendingMode); token.empty()) { + emitSyntaxError("expected lowercase blending mode enum"); + } + advanceCursorByToken(); + }; + + auto parseShadingModel = [&]() { + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readSymbol(cursor, '='); token.empty()) { + emitSyntaxError("expected equal sign"); + } + advanceCursorByToken(); + token = readWhitespace(cursor); + advanceCursorByToken(); + if (token = readShadingModel(cursor, &material.shadingModel); token.empty()) { + emitSyntaxError("expected lowercase shading enum"); + } + advanceCursorByToken(); + }; + + if (cursor[0] == 0 || cursor[0] == '#') { + ++mLineNumber; + return; + } + + if (size_t sz = readPrefix("BlendingMode"sv, cursor); sz > 0) { + advanceCursorByCount(sz); + parseBlendingMode(); + } else if (size_t sz = readPrefix("ShadingModel"sv, cursor); sz > 0) { + advanceCursorByCount(sz); + parseShadingModel(); + } else { + parseFeatureFlag(); + } + + if (cursor[0] != 0) { + emitSyntaxError("unexpected trailing character"); + } + + ++mLineNumber; +} + +FixedCapacityVector WritableArchive::serialize() const { + size_t byteCount = sizeof(ReadableArchive); + for (const auto& mat : mMaterials) { + byteCount += sizeof(ArchiveSpec); + } + size_t flaglistOffset = byteCount; + for (const auto& mat : mMaterials) { + for (const auto& pair : mat.flags) { + byteCount += sizeof(ArchiveFlag); + } + } + size_t nameOffset = byteCount; + for (const auto& mat : mMaterials) { + for (const auto& pair : mat.flags) { + byteCount += pair.first.size() + 1; + } + } + size_t filamatOffset = byteCount; + for (const auto& mat : mMaterials) { + byteCount += mat.package.size(); + } + + ReadableArchive archive; + archive.magic = 'UBER'; + archive.version = 0; + archive.specsCount = mMaterials.size(); + archive.specsOffset = sizeof(ReadableArchive); + + auto specs = FixedCapacityVector::with_capacity(mMaterials.size()); + size_t flagCount = 0; + for (const auto& mat : mMaterials) { + ArchiveSpec spec = {}; + spec.shadingModel = mat.shadingModel; + spec.blendingMode = mat.blendingMode; + spec.flagsCount = mat.flags.size(); + spec.flagsOffset = flaglistOffset + flagCount * sizeof(ArchiveFlag); + spec.packageByteCount = mat.package.size(); + spec.packageOffset = filamatOffset; + specs.push_back(spec); + filamatOffset += mat.package.size(); + flagCount += mat.flags.size(); + } + + auto flags = FixedCapacityVector::with_capacity(flagCount); + size_t charCount = 0; + for (const auto& mat : mMaterials) { + for (const auto& pair : mat.flags) { + ArchiveFlag flag; + flag.nameOffset = nameOffset + charCount; + flag.value = pair.second; + charCount += pair.first.size() + 1; + flags.push_back(flag); + } + } + + std::string flagNames(charCount, ' '); + char* flagNamesPtr = flagNames.data(); + for (const auto& mat : mMaterials) { + for (const auto& pair : mat.flags) { + memcpy(flagNamesPtr, pair.first.data(), pair.first.size() + 1); + flagNamesPtr += pair.first.size() + 1; + } + } + assert(flagNamesPtr - flagNames.data() == flagNames.size()); + + FixedCapacityVector outputBuf(byteCount); + uint8_t* writeCursor = outputBuf.data(); + memcpy(writeCursor, &archive, sizeof(archive)); + writeCursor += sizeof(archive); + memcpy(writeCursor, specs.data(), sizeof(ArchiveSpec) * specs.size()); + writeCursor += sizeof(ArchiveSpec) * specs.size(); + memcpy(writeCursor, flags.data(), sizeof(ArchiveFlag) * flags.size()); + writeCursor += sizeof(ArchiveFlag) * flags.size(); + memcpy(writeCursor, flagNames.data(), charCount); + writeCursor += charCount; + for (const auto& mat : mMaterials) { + memcpy(writeCursor, mat.package.data(), mat.package.size()); + writeCursor += mat.package.size(); + } + assert_invariant(writeCursor - outputBuf.data() == outputBuf.size()); + + FixedCapacityVector compressedBuf(ZSTD_compressBound(outputBuf.size())); + size_t zstdResult = ZSTD_compress(compressedBuf.data(), compressedBuf.size(), outputBuf.data(), + outputBuf.size(), ZSTD_maxCLevel()); + if (ZSTD_isError(zstdResult)) { + PANIC_POSTCONDITION("Error during archive compression: %s", ZSTD_getErrorName(zstdResult)); + } + + compressedBuf.resize(zstdResult); + return compressedBuf; +} + +} // namespace filament::uberz diff --git a/libs/utils/benchmark/benchmark_binary_search.cpp b/libs/utils/benchmark/benchmark_binary_search.cpp index 110a9c4088..376d59ed1d 100644 --- a/libs/utils/benchmark/benchmark_binary_search.cpp +++ b/libs/utils/benchmark/benchmark_binary_search.cpp @@ -16,14 +16,12 @@ #include "PerformanceCounters.h" -#include #include #include #include #include -#include using namespace utils; @@ -95,70 +93,5 @@ BENCHMARK_DEFINE_F(BinarySearch, stdLowerBound)(benchmark::State& state) { } } -BENCHMARK_DEFINE_F(BinarySearch, utilsLowerBound)(benchmark::State& state) { - auto first = data.begin(); - auto last = data.begin() + state.range(0); - std::vector indices = prepareItems(state.range(0)); - value_type const* ip = indices.data(); - size_t i = 0; - - { - PerformanceCounters pc(state); - for (auto _ : state) { - auto item = ip[i++ % state.range(0)]; - auto const& pos = utils::lower_bound(first, last, item); - benchmark::DoNotOptimize(pos); - } - } -} - -BENCHMARK_DEFINE_F(BinarySearch, utilsLowerBoundPOT)(benchmark::State& state) { - auto first = data.begin(); - auto last = data.begin() + state.range(0); - std::vector indices = prepareItems(state.range(0)); - value_type const* ip = indices.data(); - size_t i = 0; - - { - PerformanceCounters pc(state); - for (auto _ : state) { - auto item = ip[i++ % state.range(0)]; - auto const& pos = utils::lower_bound(first, last, item, {}, true); - benchmark::DoNotOptimize(pos); - } - } -} - -BENCHMARK_DEFINE_F(BinarySearch, utilsLowerBound4096)(benchmark::State& state) { - auto first = data.begin(); - auto last = data.begin() + 4096; - std::vector indices = prepareItems(4096); - value_type const* ip = indices.data(); - size_t i = 0; - - { - PerformanceCounters pc(state); - for (auto _ : state) { - auto item = ip[i++ % 4096]; - auto const& pos = utils::lower_bound(first, last, item, {}, true); - benchmark::DoNotOptimize(pos); - } - } -} - BENCHMARK_REGISTER_F(BinarySearch, linearSearch)->Range(2, 1<<20); BENCHMARK_REGISTER_F(BinarySearch, stdLowerBound)->Range(2, 1<<20); -BENCHMARK_REGISTER_F(BinarySearch, utilsLowerBound)->Range(2, 1<<20); -BENCHMARK_REGISTER_F(BinarySearch, utilsLowerBoundPOT) - ->Arg(2) - ->Arg(4) - ->Arg(16) - ->Arg(32) - ->Arg(64) - ->Arg(256) - ->Arg(4096) - ->Arg(32768) - ->Arg(262144) - ->Arg(1048576); - -BENCHMARK_REGISTER_F(BinarySearch, utilsLowerBound4096); diff --git a/libs/utils/include/utils/Allocator.h b/libs/utils/include/utils/Allocator.h index 90c0acb328..ba9fe198f6 100644 --- a/libs/utils/include/utils/Allocator.h +++ b/libs/utils/include/utils/Allocator.h @@ -457,6 +457,12 @@ private: void* mEnd = nullptr; }; +class NullArea { +public: + void* data() const noexcept { return nullptr; } + size_t size() const noexcept { return 0; } +}; + } // namespace AreaPolicy // ------------------------------------------------------------------------------------------------ diff --git a/libs/utils/include/utils/CString.h b/libs/utils/include/utils/CString.h index da809d11cf..09f3d2ca6a 100644 --- a/libs/utils/include/utils/CString.h +++ b/libs/utils/include/utils/CString.h @@ -79,7 +79,7 @@ public: using const_pointer = const value_type*; using const_iterator = const value_type*; - constexpr StaticString() noexcept = default; + constexpr StaticString() noexcept {} // NOLINT(modernize-use-equals-default), Ubuntu compiler bug // initialization from a string literal template @@ -149,6 +149,14 @@ public: size_type getHash() const noexcept { return mHash; } + struct Hasher { + typedef StaticString argument_type; + typedef size_t result_type; + result_type operator()(const argument_type& s) const noexcept { + return s.getHash(); + } + }; + private: const_pointer mString = nullptr; size_type mLength = 0; @@ -200,7 +208,7 @@ public: using iterator = value_type*; using const_iterator = const value_type*; - CString() noexcept = default; + CString() noexcept {} // NOLINT(modernize-use-equals-default), Ubuntu compiler bug // Allocates memory and appends a null. This constructor can be used to hold arbitrary data // inside the string (i.e. it can contain nulls or non-ASCII encodings). @@ -220,7 +228,7 @@ public: : CString(other, N - 1) { } - CString(StaticString const& s) : CString(s.c_str(), s.size()) {} + CString(StaticString const& s) : CString(s.c_str(), s.size()) {} // NOLINT(google-explicit-constructor) CString(const CString& rhs); @@ -309,11 +317,19 @@ public: } // placement new declared as "throw" to avoid the compiler's null-check - inline void* operator new(size_t size, void* ptr) { + inline void* operator new(size_t, void* ptr) { assert(ptr); return ptr; } + struct Hasher : private hashCStrings { + typedef CString argument_type; + typedef size_t result_type; + result_type operator()(const argument_type& s) const noexcept { + return hashCStrings::operator()(s.c_str()); + } + }; + private: struct Data { size_type length; @@ -365,34 +381,4 @@ CString to_string(T value) noexcept; } // namespace utils -// FIXME: how could we not include this one? -// needed for std::hash, since implementation is inline, this would not cause -// binaries incompatibilities if another STL version was used. -#include - -namespace std { - -//! \privatesection -template<> -struct hash { - typedef utils::CString argument_type; - typedef size_t result_type; - utils::hashCStrings hasher; - size_t operator()(const utils::CString& s) const noexcept { - return hasher(s.c_str()); - } -}; - -//! \privatesection -template<> -struct hash { - typedef utils::StaticString argument_type; - typedef size_t result_type; - size_t operator()(const utils::StaticString& s) const noexcept { - return s.getHash(); - } -}; - -} // namespace std - #endif // TNT_UTILS_CSTRING_H diff --git a/libs/utils/include/utils/Entity.h b/libs/utils/include/utils/Entity.h index 74f417aaaa..58fdf606ae 100644 --- a/libs/utils/include/utils/Entity.h +++ b/libs/utils/include/utils/Entity.h @@ -19,9 +19,6 @@ #include -// FIXME: could we get rid of -#include // for std::hash - #include #include @@ -30,7 +27,7 @@ namespace utils { class UTILS_PUBLIC Entity { public: // this can be used to create an array of to-be-filled entities (see create()) - Entity() noexcept = default; + Entity() noexcept { } // NOLINT(modernize-use-equals-default), Ubuntu compiler bug // Entities can be copied Entity(const Entity& e) noexcept = default; @@ -68,10 +65,17 @@ public: return Entity{ Type(identity) }; } + struct Hasher { + typedef Entity argument_type; + typedef size_t result_type; + result_type operator()(argument_type const& e) const { + return e.getId(); + } + }; + private: friend class EntityManager; friend class EntityManagerImpl; - friend struct std::hash; using Type = uint32_t; explicit Entity(Type identity) noexcept : mIdentity(identity) { } @@ -81,18 +85,4 @@ private: } // namespace utils - -namespace std { - -template<> -struct hash { - typedef utils::Entity argument_type; - typedef size_t result_type; - result_type operator()(argument_type const& e) const { - return e.getId(); - } -}; - -} // namespace std - #endif // TNT_UTILS_ENTITY_H diff --git a/libs/utils/include/utils/Hash.h b/libs/utils/include/utils/Hash.h index 8d0bd9ff16..f17dfc8655 100644 --- a/libs/utils/include/utils/Hash.h +++ b/libs/utils/include/utils/Hash.h @@ -22,8 +22,7 @@ #include #include -namespace utils { -namespace hash { +namespace utils::hash { // Hash function that takes an arbitrary swath of word-aligned data. inline uint32_t murmur3(const uint32_t* key, size_t wordCount, uint32_t seed) noexcept { @@ -99,7 +98,6 @@ inline void combine_fast(size_t& seed, const T& v) noexcept { seed ^= hasher(v) << 1u; } -} // namespace hash -} // namespace utils +} // namespace utils::hash #endif // TNT_UTILS_HASH_H diff --git a/libs/utils/include/utils/NameComponentManager.h b/libs/utils/include/utils/NameComponentManager.h index 62201d113a..9e31e4618d 100644 --- a/libs/utils/include/utils/NameComponentManager.h +++ b/libs/utils/include/utils/NameComponentManager.h @@ -17,40 +17,19 @@ #ifndef TNT_UTILS_NAMECOMPONENTMANAGER_H #define TNT_UTILS_NAMECOMPONENTMANAGER_H -#include -#include -#include - #include +#include #include #include #include -#include +#include +#include namespace utils { class EntityManager; -namespace details { -class SafeString { -public: - SafeString() noexcept = default; - explicit SafeString(const char* str) noexcept : mCStr(strdup(str)) { } - SafeString(SafeString&& rhs) noexcept : mCStr(rhs.mCStr) { rhs.mCStr = nullptr; } - SafeString& operator=(SafeString&& rhs) noexcept { - std::swap(mCStr, rhs.mCStr); - return *this; - } - ~SafeString() { free((void*)mCStr); } - const char* c_str() const noexcept { return mCStr; } - -private: - char const* mCStr = nullptr; -}; -} // namespace details - - /** * \class NameComponentManager NameComponentManager.h utils/NameComponentManager.h * \brief Allows clients to associate string labels with entities. @@ -69,7 +48,7 @@ private: * printf("%s\n", names->getName(names->getInstance(myEntity)); * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -class UTILS_PUBLIC NameComponentManager : public SingleInstanceComponentManager { +class UTILS_PUBLIC NameComponentManager : public SingleInstanceComponentManager { public: using Instance = EntityInstance; @@ -93,7 +72,7 @@ public: * @return Non-zero handle if the entity has a name component, 0 otherwise. */ Instance getInstance(Entity e) const noexcept { - return Instance(SingleInstanceComponentManager::getInstance(e)); + return { SingleInstanceComponentManager::getInstance(e) }; } /*! \cond PRIVATE */ diff --git a/libs/utils/include/utils/SingleInstanceComponentManager.h b/libs/utils/include/utils/SingleInstanceComponentManager.h index 574d14a745..e728d6ee90 100644 --- a/libs/utils/include/utils/SingleInstanceComponentManager.h +++ b/libs/utils/include/utils/SingleInstanceComponentManager.h @@ -256,7 +256,7 @@ protected: private: // maps an entity to an instance index - tsl::robin_map mInstanceMap; + tsl::robin_map mInstanceMap; default_random_engine mRng; }; diff --git a/libs/utils/include/utils/Stopwatch.h b/libs/utils/include/utils/Stopwatch.h new file mode 100644 index 0000000000..fcd3051232 --- /dev/null +++ b/libs/utils/include/utils/Stopwatch.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_UTILS_STOPWATCH_H +#define TNT_UTILS_STOPWATCH_H + +#include + +#include +#include + +#include + +namespace utils { + +/* + * A very basic Stopwatch class + */ + +template +class Stopwatch { +public: + using duration = typename Clock::duration; + using time_point = typename Clock::time_point; + +private: + time_point mStart; + duration mMinLap = std::numeric_limits::max(); + duration mMaxLap{}; + std::chrono::duration mAvgLap{}; + size_t mCount = 0; + const char* mName = nullptr; + +public: + // Create a Stopwatch with a name and clock + explicit Stopwatch(const char* name) noexcept: mName(name) {} + + // Logs min/avg/max lap time + ~Stopwatch() noexcept; + + // start the stopwatch + inline void start() noexcept { + mStart = Clock::now(); + } + + // stop the stopwatch + inline void stop() noexcept { + auto d = Clock::now() - mStart; + mMinLap = std::min(mMinLap, d); + mMaxLap = std::max(mMaxLap, d); + mAvgLap = (mAvgLap * mCount + d) / (mCount + 1); + mCount++; + } + + // get the minimum lap time recorded + duration getMinLapTime() const noexcept { return mMinLap; } + + // get the maximum lap time recorded + duration getMaxLapTime() const noexcept { return mMaxLap; } + + // get the average lap time + duration getAverageLapTime() const noexcept { return mAvgLap; } +}; + +template +Stopwatch::~Stopwatch() noexcept { + slog.d << "Stopwatch \"" << mName << "\" : [" + << mMinLap.count() << ", " + << std::chrono::duration_cast(mAvgLap).count() << ", " + << mMaxLap.count() << "] ns" << io::endl; +} + +/* + * AutoStopwatch can be used to start and stop a Stopwatch automatically + * when entering and exiting a scope. + */ +template +class AutoStopwatch { + Stopwatch& stopwatch; +public: + inline explicit AutoStopwatch(Stopwatch& stopwatch) noexcept: stopwatch(stopwatch) { + stopwatch.start(); + } + + inline ~AutoStopwatch() noexcept { stopwatch.stop(); } +}; + +} // namespace utils + +#endif // TNT_UTILS_STOPWATCH_H diff --git a/libs/utils/include/utils/StructureOfArrays.h b/libs/utils/include/utils/StructureOfArrays.h index dd879a6fa3..41cd8b890c 100644 --- a/libs/utils/include/utils/StructureOfArrays.h +++ b/libs/utils/include/utils/StructureOfArrays.h @@ -19,7 +19,6 @@ #include // note: this is safe, see how std::array is used below (inline / private) #include -#include #include #include diff --git a/libs/utils/include/utils/algorithm.h b/libs/utils/include/utils/algorithm.h index 72240c2730..6d12ffc63a 100644 --- a/libs/utils/include/utils/algorithm.h +++ b/libs/utils/include/utils/algorithm.h @@ -19,7 +19,6 @@ #include -#include // for std::less #include // for std::enable_if #include @@ -168,74 +167,6 @@ T log2i(T x) noexcept { return (sizeof(x) * 8 - 1u) - clz(x); } -/* - * branch-less version of std::lower_bound and std::upper_bound. - * These versions are intended to be fully inlined, which only happens when the size - * of the array is known at compile time. This code also performs better if the - * array is a power-of-two in size. - * - * These code works even if the conditions above are not met, and becomes a less-branches - * algorithm instead of a branch-less one! - */ - -template> -inline UTILS_PUBLIC -RandomAccessIterator lower_bound( - RandomAccessIterator first, RandomAccessIterator last, const T& value, - COMPARE comp = std::less(), - bool assume_power_of_two = false) { - size_t len = last - first; - - if (!assume_power_of_two) { - // handle non power-of-two sized arrays. If it's POT, the next line is a no-op - // and gets optimized out if the size is known at compile time. - len = 1u << (31 - clz(uint32_t(len))); // next power of two length / 2 - size_t difference = (last - first) - len; - // If len was already a POT, then difference will be 0. - // We need to explicitly check this case to avoid dereferencing past the end of the array - first += !difference || comp(first[len], value) ? difference : 0; - } - - while (len) { - // The number of repetitions here doesn't affect the result. We manually unroll the loop - // twice, to guarantee we have at least two iterations without branches (for the case - // where the size is not known at compile time - first += comp(first[len >>= 1u], value) ? len : 0; - first += comp(first[len >>= 1u], value) ? len : 0; - } - first += comp(*first, value); - return first; -} - -template> -inline UTILS_PUBLIC -RandomAccessIterator upper_bound( - RandomAccessIterator first, RandomAccessIterator last, - const T& value, COMPARE comp = std::less(), - bool assume_power_of_two = false) { - size_t len = last - first; - - if (!assume_power_of_two) { - // handle non power-of-two sized arrays. If it's POT, the next line is a no-op - // and gets optimized out if the size is known at compile time. - len = 1u << (31 - clz(uint32_t(len))); // next power of two length / 2 - size_t difference = (last - first) - len; - // If len was already a POT, then difference will be 0. - // We need to explicitly check this case to avoid dereferencing past the end of the array - first += !difference || comp(value, first[len]) ? 0 : difference; - } - - while (len) { - // The number of repetitions here doesn't affect the result. We manually unroll the loop - // twice, to guarantee we have at least two iterations without branches (for the case - // where the size is not known at compile time - first += !comp(value, first[len >>= 1u]) ? len : 0; - first += !comp(value, first[len >>= 1u]) ? len : 0; - } - first += !comp(value, *first); - return first; -} - template inline UTILS_PUBLIC RandomAccessIterator partition_point( diff --git a/libs/utils/src/Allocator.cpp b/libs/utils/src/Allocator.cpp index 7489bf24e1..2d7a8fcbe9 100644 --- a/libs/utils/src/Allocator.cpp +++ b/libs/utils/src/Allocator.cpp @@ -135,13 +135,15 @@ void TrackingPolicy::HighWatermark::onAlloc( } TrackingPolicy::HighWatermark::~HighWatermark() noexcept { + const size_t wm = mHighWaterMark; if (mSize > 0) { - size_t wm = mHighWaterMark; size_t wmpct = wm / (mSize / 100); if (wmpct > 80) { slog.d << mName << " arena: High watermark " << wm / 1024 << " KiB (" << wmpct << "%)" << io::endl; } + } else { + slog.d << mName << " arena: High watermark " << wm / 1024 << " KiB" << io::endl; } } diff --git a/libs/utils/src/EntityManagerImpl.h b/libs/utils/src/EntityManagerImpl.h index f8ea374b50..e9c7ef1699 100644 --- a/libs/utils/src/EntityManagerImpl.h +++ b/libs/utils/src/EntityManagerImpl.h @@ -178,7 +178,7 @@ private: tsl::robin_set mListeners; #if FILAMENT_UTILS_TRACK_ENTITIES - tsl::robin_map mDebugActiveEntities; + tsl::robin_map mDebugActiveEntities; #endif }; diff --git a/libs/utils/src/NameComponentManager.cpp b/libs/utils/src/NameComponentManager.cpp index b3fd2daedc..98bb3839a3 100644 --- a/libs/utils/src/NameComponentManager.cpp +++ b/libs/utils/src/NameComponentManager.cpp @@ -28,7 +28,7 @@ NameComponentManager::~NameComponentManager() = default; void NameComponentManager::setName(Instance instance, const char* name) noexcept { if (instance) { - elementAt(instance) = details::SafeString{ name }; + elementAt(instance) = CString{ name }; } } diff --git a/libs/utils/test/test_CString.cpp b/libs/utils/test/test_CString.cpp index 7333e69e1c..16ef2c2baa 100644 --- a/libs/utils/test/test_CString.cpp +++ b/libs/utils/test/test_CString.cpp @@ -36,7 +36,7 @@ TEST(StaticString, hash) { EXPECT_NE(a.getHash(), c.getHash()); EXPECT_NE(b.getHash(), c.getHash()); - std::hash ha; + StaticString::Hasher ha; EXPECT_EQ(ha(a), a.getHash()); } diff --git a/libs/utils/test/test_algorithm.cpp b/libs/utils/test/test_algorithm.cpp index 97f4401ad8..fa36aff0dc 100644 --- a/libs/utils/test/test_algorithm.cpp +++ b/libs/utils/test/test_algorithm.cpp @@ -138,91 +138,6 @@ TEST(AlgorithmTest, popcount) { } } -TEST(AlgorithmTest, UpperBounds) { - int* r; - int array[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - - for (int i = 0; i < 8; i++) { - r = utils::upper_bound(std::begin(array), std::end(array), array[i]); - EXPECT_EQ(i + 1, r - std::begin(array)); - } - - r = utils::upper_bound(std::begin(array), std::end(array), 100); - EXPECT_EQ(std::end(array), r); - - r = utils::upper_bound(std::begin(array), std::end(array), -1); - EXPECT_EQ(std::begin(array), r); - - - int array2[8] = { 0, 0, 0, 1, 1, 1, 2, 2 }; - r = utils::upper_bound(std::begin(array2), std::end(array2), 0); - EXPECT_EQ(std::begin(array2) + 3, r); - - r = utils::upper_bound(std::begin(array2), std::end(array2), 1); - EXPECT_EQ(std::begin(array2) + 6, r); - - r = utils::upper_bound(std::begin(array2), std::end(array2), 2); - EXPECT_EQ(std::end(array2), r); - - - int arrayNPotEven[10] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - for (int i = 0; i < 10; i++) { - r = utils::upper_bound(std::begin(arrayNPotEven), std::end(arrayNPotEven), arrayNPotEven[i]); - EXPECT_EQ(i + 1, r - std::begin(arrayNPotEven)); - } - - int arrayNPotOdd[15] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; - for (int i = 0; i < 15; i++) { - r = utils::upper_bound(std::begin(arrayNPotOdd), std::end(arrayNPotOdd), arrayNPotOdd[i]); - EXPECT_EQ(i + 1, r - std::begin(arrayNPotOdd)); - } - - std::array arr = {{1, 10, 15, 50}}; - EXPECT_EQ(50, *utils::upper_bound(arr.begin(), arr.end(), 15)); -} - -TEST(AlgorithmTest, LowerBounds) { - int* r; - int array[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; - - for (int i = 0; i < 8; i++) { - r = utils::lower_bound(std::begin(array), std::end(array), array[i]); - EXPECT_EQ(i, r - std::begin(array)); - } - - r = utils::lower_bound(std::begin(array), std::end(array), 100); - EXPECT_EQ(std::end(array), r); - - r = utils::lower_bound(std::begin(array), std::end(array), -1); - EXPECT_EQ(std::begin(array), r); - - - int array2[8] = { 0, 0, 0, 1, 1, 1, 2, 2 }; - r = utils::lower_bound(std::begin(array2), std::end(array2), 0); - EXPECT_EQ(std::begin(array2), r); - - r = utils::lower_bound(std::begin(array2), std::end(array2), 1); - EXPECT_EQ(std::begin(array2) + 3, r); - - r = utils::lower_bound(std::begin(array2), std::end(array2), 2); - EXPECT_EQ(std::begin(array2) + 6, r); - - int arrayNPotEven[10] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - for (int i = 0; i < 10; i++) { - r = utils::lower_bound(std::begin(arrayNPotEven), std::end(arrayNPotEven), arrayNPotEven[i]); - EXPECT_EQ(i, r - std::begin(arrayNPotEven)); - } - - int arrayNPotOdd[15] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; - for (int i = 0; i < 15; i++) { - r = utils::lower_bound(std::begin(arrayNPotOdd), std::end(arrayNPotOdd), arrayNPotOdd[i]); - EXPECT_EQ(i, r - std::begin(arrayNPotOdd)); - } - - std::array arr = {{1, 10, 15, 50}}; - EXPECT_EQ(15, *utils::lower_bound(arr.begin(), arr.end(), 15)); -} - TEST(AlgorithmTest, Partition) { int* r; int array[8] = { 2, 5, 4, 8, 9, 9, 9, 9 }; diff --git a/libs/viewer/include/viewer/ViewerGui.h b/libs/viewer/include/viewer/ViewerGui.h index 348b0870ab..8192c32ee7 100644 --- a/libs/viewer/include/viewer/ViewerGui.h +++ b/libs/viewer/include/viewer/ViewerGui.h @@ -36,6 +36,7 @@ #include #include +#include #include namespace filagui { diff --git a/tools/uberz/CMakeLists.txt b/tools/uberz/CMakeLists.txt new file mode 100644 index 0000000000..aadc01c1b0 --- /dev/null +++ b/tools/uberz/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.19) +project(uberz) + +set(TARGET uberz) + +# ================================================================================================== +# Source files +# ================================================================================================== +set(SRCS + src/main.cpp) + +# ================================================================================================== +# Target definitions +# ================================================================================================== +add_executable(${TARGET} ${SRCS}) +target_link_libraries(${TARGET} PRIVATE utils getopt uberzlib) + +# ================================================================================================= +# Licenses +# ================================================================================================== +set(MODULE_LICENSES getopt) +set(GENERATION_ROOT ${CMAKE_CURRENT_BINARY_DIR}/generated) +list_licenses(${GENERATION_ROOT}/licenses/licenses.inc ${MODULE_LICENSES}) +target_include_directories(${TARGET} PRIVATE ${GENERATION_ROOT}) + +# ================================================================================================== +# Installation +# ================================================================================================== +install(TARGETS ${TARGET} RUNTIME DESTINATION bin) diff --git a/tools/uberz/src/main.cpp b/tools/uberz/src/main.cpp new file mode 100644 index 0000000000..726c928ac1 --- /dev/null +++ b/tools/uberz/src/main.cpp @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include +#include +#include + +#include + +#include + +using namespace std; +using namespace utils; +using namespace filament::uberz; + +static std::string g_outputFile = "materials.uberz"; +static bool g_quietMode = false; +static bool g_verboseMode = false; + +static const char* USAGE = R"TXT( +UBERZ aggregates and compresses a set of filamat files into a single archive file. It includes +metadata that specifies the feature set that each material supports. By default, it generates +a file called "materials.uberz" but this can be customized with -o. + +Usage: + UBERZ [options] ... + +For each src_name, UBERZ looks for "src_name.filamat" and "src_name.spec" in the current +working directory. If either of these files do not exist, an error is reported. Each +pair of filamat/spec files corresponds to a material in the generated archive. + +For more information on the format of the spec file, see the gltfio README. + +Options: + --help, -h + Print this message + --license, -L + Print copyright and license information + --output=filename, -o filename + Specify a custom filename. + --quiet, -q + Suppress console output +)TXT"; + +static void printUsage(const char* name) { + std::string execName(Path(name).getName()); + const std::string from("UBERZ"); + std::string usage(USAGE); + for (size_t pos = usage.find(from); pos != std::string::npos; pos = usage.find(from, pos)) { + usage.replace(pos, from.length(), execName); + } + puts(usage.c_str()); +} + +static void license() { + static const char *license[] = { + #include "licenses/licenses.inc" + nullptr + }; + + const char **p = &license[0]; + while (*p) + std::cout << *p++ << std::endl; +} + +static int handleArguments(int argc, char* argv[]) { + static constexpr const char* OPTSTR = "hLqvo:"; + static const struct option OPTIONS[] = { + { "help", no_argument, 0, 'h' }, + { "license", no_argument, 0, 'L' }, + { "quiet", no_argument, 0, 'q' }, + { "verbose", no_argument, 0, 'v' }, + { "output", required_argument, 0, 'o' }, + { 0, 0, 0, 0 } // termination of the option list + }; + + int opt; + int optionIndex = 0; + + while ((opt = getopt_long(argc, argv, OPTSTR, OPTIONS, &optionIndex)) >= 0) { + std::string arg(optarg ? optarg : ""); + switch (opt) { + default: + case 'h': + printUsage(argv[0]); + exit(0); + case 'L': + license(); + exit(0); + case 'o': + g_outputFile = optarg; + break; + case 'q': + g_quietMode = true; + break; + case 'v': + g_verboseMode = true; + break; + } + } + + return optind; +} + +static size_t getFileSize(const char* filename) { + std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary); + return in.tellg(); +}; + +int main(int argc, char* argv[]) { + const int optionIndex = handleArguments(argc, argv); + const int numArgs = argc - optionIndex; + if (numArgs < 1) { + printUsage(argv[0]); + return 1; + } + + WritableArchive archive(argc - optionIndex); + + for (int argIndex = optionIndex; argIndex < argc; ++argIndex) { + std::string name(argv[argIndex]); + const Path filamatPath(name + ".filamat"); + if (!filamatPath.exists()) { + cerr << "Unable to open " << filamatPath << endl; + exit(1); + } + const Path specPath(name + ".spec"); + if (!specPath.exists()) { + cerr << "Unable to open " << specPath << endl; + exit(1); + } + + const size_t filamatSize = getFileSize(filamatPath.c_str()); + FixedCapacityVector filamatBuffer(filamatSize); + std::ifstream in(filamatPath.c_str(), std::ifstream::in | std::ifstream::binary); + if (!in.read((char*) filamatBuffer.data(), filamatSize)) { + cerr << "Unable to consume " << filamatPath << endl; + exit(1); + } + + // This tool is often invoked by CMake, so in verbose mode we print an index to stderr. + // This allows us to see the spec index associated with each file for diagnostic purposes. + if (g_verboseMode) { + fprintf(stderr, "uberz %2d %s\n", argIndex - optionIndex, filamatPath.getName().c_str()); + } + + archive.addMaterial(name.c_str(), filamatBuffer.data(), filamatSize); + + std::string specLine; + ifstream specStream(specPath.c_str()); + while (std::getline(specStream, specLine)) { + archive.addSpecLine(specLine.c_str()); + } + } + + FixedCapacityVector binBuffer = archive.serialize(); + + ofstream binStream(g_outputFile, ios::binary); + if (!binStream) { + cerr << "Unable to open " << g_outputFile << endl; + exit(1); + } + binStream.write((const char*) binBuffer.data(), binBuffer.size()); + binStream.close(); + + return 0; +} diff --git a/web/filament-js/package.json b/web/filament-js/package.json index f0795a9b37..b1b4bd9375 100644 --- a/web/filament-js/package.json +++ b/web/filament-js/package.json @@ -1,6 +1,6 @@ { "name": "filament", - "version": "1.23.1", + "version": "1.23.2", "description": "Real-time physically based rendering engine", "main": "filament.js", "module": "filament.js",