Scarfski/filament
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for KHR_texture_basisu.

Browse Source 
This adds a new implementation of the TextureProvider interface called
Ktx2Provider.

Tested using the KTX2 variant of the StainedGlassLamp model in the
Khronos samples repo.

Tested on WebGL 2.0 (Chrome v100), Android (Pixel 6 Pro), and Desktop
(Metal, OpenGL, and Vulkan via MoltenVK).
This commit is contained in:
Philip Rideout
2022-04-29 13:48:49 -07:00
parent f498e08d1e
commit f035655f5b
15 changed files with 335 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -177,6 +177,7 @@ steps:
- [x] KHR_materials_variants
- [x] KHR_materials_volume
- [x] KHR_mesh_quantization
- [x] KHR_texture_basisu
- [x] KHR_texture_transform

1
android/gltfio-android/CMakeLists.txt
View File

@@ -58,6 +58,7 @@ set(GLTFIO_SRCS
${GLTFIO_DIR}/src/FFilamentInstance.h
${GLTFIO_DIR}/src/FilamentInstance.cpp
${GLTFIO_DIR}/src/GltfEnums.h
${GLTFIO_DIR}/src/Ktx2Provider.cpp
${GLTFIO_DIR}/src/MaterialProvider.cpp
${GLTFIO_DIR}/src/ResourceLoader.cpp
${GLTFIO_DIR}/src/StbProvider.cpp

9
android/gltfio-android/src/main/cpp/ResourceLoader.cpp
View File

@@ -117,12 +117,19 @@ Java_com_google_android_filament_gltfio_ResourceLoader_nAsyncCancelLoad(JNIEnv*,
}
extern "C" JNIEXPORT jlong JNICALL
Java_com_google_android_filament_gltfio_ResourceLoader_nCreateTextureProvider(JNIEnv*, jclass,
Java_com_google_android_filament_gltfio_ResourceLoader_nCreateStbProvider(JNIEnv*, jclass,
jlong nativeEngine) {
Engine* engine = (Engine*) nativeEngine;
return (jlong) createStbProvider(engine);
}
extern "C" JNIEXPORT jlong JNICALL
Java_com_google_android_filament_gltfio_ResourceLoader_nCreateKtx2Provider(JNIEnv*, jclass,
jlong nativeEngine) {
Engine* engine = (Engine*) nativeEngine;
return (jlong) createKtx2Provider(engine);
}
extern "C" JNIEXPORT void JNICALL
Java_com_google_android_filament_gltfio_ResourceLoader_nDestroyTextureProvider(JNIEnv*, jclass,
jlong nativeProvider) {

25
android/gltfio-android/src/main/java/com/google/android/filament/gltfio/ResourceLoader.java
View File

@@ -35,7 +35,8 @@ import java.nio.Buffer;
*/
public class ResourceLoader {
private final long mNativeObject;
private final long mNativeProvider;
private final long mNativeStbProvider;
private final long mNativeKtx2Provider;
/**
* Constructs a resource loader tied to the given Filament engine.
@@ -47,9 +48,11 @@ public class ResourceLoader {
public ResourceLoader(@NonNull Engine engine) {
long nativeEngine = engine.getNativeObject();
mNativeObject = nCreateResourceLoader(nativeEngine, false, false, false);
mNativeProvider = nCreateTextureProvider(nativeEngine);
nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider);
nAddTextureProvider(mNativeObject, "image/png", mNativeProvider);
mNativeStbProvider = nCreateStbProvider(nativeEngine);
mNativeKtx2Provider = nCreateKtx2Provider(nativeEngine);
nAddTextureProvider(mNativeObject, "image/jpeg", mNativeStbProvider);
nAddTextureProvider(mNativeObject, "image/png", mNativeStbProvider);
nAddTextureProvider(mNativeObject, "image/ktx2", mNativeKtx2Provider);
}
/**
@@ -67,9 +70,11 @@ public class ResourceLoader {
long nativeEngine = engine.getNativeObject();
mNativeObject = nCreateResourceLoader(nativeEngine, normalizeSkinningWeights,
recomputeBoundingBoxes, ignoreBindTransform);
mNativeProvider = nCreateTextureProvider(nativeEngine);
nAddTextureProvider(mNativeObject, "image/jpeg", mNativeProvider);
nAddTextureProvider(mNativeObject, "image/png", mNativeProvider);
mNativeStbProvider = nCreateStbProvider(nativeEngine);
mNativeKtx2Provider = nCreateKtx2Provider(nativeEngine);
nAddTextureProvider(mNativeObject, "image/jpeg", mNativeStbProvider);
nAddTextureProvider(mNativeObject, "image/png", mNativeStbProvider);
nAddTextureProvider(mNativeObject, "image/ktx2", mNativeKtx2Provider);
}
/**
@@ -77,7 +82,8 @@ public class ResourceLoader {
*/
public void destroy() {
nDestroyResourceLoader(mNativeObject);
nDestroyTextureProvider(mNativeProvider);
nDestroyTextureProvider(mNativeStbProvider);
nDestroyTextureProvider(mNativeKtx2Provider);
}
/**
@@ -187,7 +193,8 @@ public class ResourceLoader {
private static native void nAsyncUpdateLoad(long nativeLoader);
private static native void nAsyncCancelLoad(long nativeLoader);
private static native long nCreateTextureProvider(long nativeEngine);
private static native long nCreateStbProvider(long nativeEngine);
private static native long nCreateKtx2Provider(long nativeEngine);
private static native void nAddTextureProvider(long nativeLoader, String url, long nativeProvider);
private static native void nDestroyTextureProvider(long nativeProvider);
}

14
ios/samples/gltf-viewer/gltf-viewer/FILModelView.mm
View File

@@ -79,7 +79,8 @@ const float kSensitivity = 100.0f;
ResourceLoader* _resourceLoader;
Manipulator<float>* _manipulator;
TextureProvider* _decoder;
TextureProvider* _stbDecoder;
TextureProvider* _ktxDecoder;
FilamentAsset* _asset;
@@ -131,9 +132,11 @@ const float kSensitivity = 100.0f;
_assetLoader = AssetLoader::create({_engine, _materialProvider, ncm, &em});
_resourceLoader = new ResourceLoader(
{.engine = _engine, .normalizeSkinningWeights = true, .recomputeBoundingBoxes = false});
_decoder = createStbProvider(_engine);
_resourceLoader->addTextureProvider("image/png", _decoder);
_resourceLoader->addTextureProvider("image/jpeg", _decoder);
_stbDecoder = createStbProvider(_engine);
_ktxDecoder = createKtx2Provider(_engine);
_resourceLoader->addTextureProvider("image/png", _stbDecoder);
_resourceLoader->addTextureProvider("image/jpeg", _stbDecoder);
_resourceLoader->addTextureProvider("image/ktx2", _ktxDecoder);
_manipulator =
Manipulator<float>::Builder().orbitHomePosition(0.0f, 0.0f, 4.0f).build(Mode::ORBIT);
@@ -252,7 +255,8 @@ const float kSensitivity = 100.0f;
[self destroyModel];
delete _manipulator;
delete _decoder;
delete _stbDecoder;
delete _ktxDecoder;
_materialProvider->destroyMaterials();
delete _materialProvider;

1
libs/gltfio/CMakeLists.txt
View File

@@ -30,6 +30,7 @@ set(SRCS
src/FFilamentInstance.h
src/FilamentInstance.cpp
src/GltfEnums.h
src/Ktx2Provider.cpp
src/MaterialProvider.cpp
src/ResourceLoader.cpp
src/StbProvider.cpp

6
libs/gltfio/include/gltfio/TextureProvider.h
View File

@@ -169,6 +169,12 @@ public:
*/
TextureProvider* createStbProvider(filament::Engine* engine);
/**
* Creates a decoder that can handle certain types of "image/ktx2" content as specified in
* the KHR_texture_basisu specification.
*/
TextureProvider* createKtx2Provider(filament::Engine* engine);
} // namespace gltfio
#endif // GLTFIO_TEXTUREPROVIDER_H

11
libs/gltfio/src/AssetLoader.cpp
View File

@@ -1305,13 +1305,10 @@ MaterialInstance* FAssetLoader::createMaterialInstance(const cgltf_data* srcAsse
void FAssetLoader::addTextureBinding(MaterialInstance* materialInstance, const char* parameterName,
const cgltf_texture* srcTexture, bool srgb) {
if (!srcTexture->image) {
if (srcTexture->basisu_image) {
const char* name = srcTexture->name ? srcTexture->name : srcTexture->basisu_image->uri;
slog.w << "BasisU is not yet supported (" << name << ")." << io::endl;
} else {
slog.w << "Texture is missing image (" << srcTexture->name << ")." << io::endl;
}
if (!srcTexture->image && !srcTexture->basisu_image) {
#ifndef NDEBUG
slog.w << "Texture is missing image (" << srcTexture->name << ")." << io::endl;
#endif
return;
}
TextureSampler dstSampler;

253
libs/gltfio/src/Ktx2Provider.cpp Normal file
View File

@@ -0,0 +1,253 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gltfio/TextureProvider.h>
#include <string>
#include <vector>
#include <utils/JobSystem.h>
#include <filament/Engine.h>
#include <filament/Texture.h>
#include <ktxreader/Ktx2Reader.h>
using namespace filament;
using namespace utils;
using std::atomic;
using std::vector;
using std::unique_ptr;
namespace gltfio {
class Ktx2Provider final : public TextureProvider {
public:
Ktx2Provider(Engine* engine);
~Ktx2Provider();
Texture* pushTexture(const uint8_t* data, size_t byteCount,
const char* mimeType, uint64_t flags) final;
Texture* popTexture() final;
void updateQueue() final;
void waitForCompletion() final;
void cancelDecoding() final;
const char* getPushMessage() const final;
const char* getPopMessage() const final;
size_t getPushedCount() const final { return mPushedCount; }
size_t getPoppedCount() const final { return mPoppedCount; }
size_t getDecodedCount() const final { return mDecodedCount; }
private:
enum class QueueItemState {
TRANSCODING, // Texture has been pushed, mipmap levels are not yet complete.
READY, // Mipmap levels are available but texture has not been popped yet.
POPPED, // Client has popped the texture from the queue.
};
enum class TranscoderState {
NOT_STARTED,
ERROR,
SUCCESS,
};
struct QueueItem {
ktxreader::Ktx2Reader::Async* async;
QueueItemState state;
atomic<TranscoderState> transcoderState;
JobSystem::Job* job;
};
void transcodeSingleTexture();
size_t mPushedCount = 0;
size_t mPoppedCount = 0;
size_t mDecodedCount = 0;
vector<unique_ptr<QueueItem> > mQueueItems;
JobSystem::Job* mDecoderRootJob;
std::string mRecentPushMessage;
std::string mRecentPopMessage;
std::unique_ptr<ktxreader::Ktx2Reader> mKtxReader;
Engine* const mEngine;
};
Texture* Ktx2Provider::pushTexture(const uint8_t* data, size_t byteCount,
const char* mimeType, FlagBits flags) {
using InternalFormat = Texture::InternalFormat;
using TransferFunction = ktxreader::Ktx2Reader::TransferFunction;
const FlagBits sRGB = FlagBits(Flags::sRGB);
auto async = mKtxReader->asyncCreate(data, byteCount,
(flags & sRGB) ? TransferFunction::sRGB : TransferFunction::LINEAR);
if (async == nullptr) {
mRecentPushMessage = "Unable to build Texture object.";
return nullptr;
}
mRecentPushMessage.clear();
QueueItem* item = mQueueItems.emplace_back(new QueueItem).get();
++mPushedCount;
item->async = async;
item->state = QueueItemState::TRANSCODING;
item->transcoderState.store(TranscoderState::NOT_STARTED);
// On single threaded systems, it is usually fine to create jobs because the job system will
// simply execute serially. However in our case, we wish to amortize the decoder cost across
// several frames, so we instead use the updateQueue() method to perform decoding.
if constexpr (!UTILS_HAS_THREADING) {
item->job = nullptr;
return async->getTexture();
}
JobSystem* js = &mEngine->getJobSystem();
item->job = jobs::createJob(*js, mDecoderRootJob, [this, item] {
using Result = ktxreader::Ktx2Reader::Result;
const bool success = Result::SUCCESS == item->async->doTranscoding();
item->transcoderState.store(success ? TranscoderState::SUCCESS : TranscoderState::ERROR);
});
js->runAndRetain(item->job);
return async->getTexture();
}
Texture* Ktx2Provider::popTexture() {
// We don't bother shrinking the mQueueItems vector here, instead we periodically clean it up in
// the updateQueue method, since popTexture is typically called more frequently. Textures
// can become ready in non-deterministic order due to concurrency.
for (auto& item : mQueueItems) {
if (item->state == QueueItemState::READY) {
item->state = QueueItemState::POPPED;
++mPoppedCount;
const TranscoderState state = item->transcoderState.load();
if (state != TranscoderState::SUCCESS) {
mRecentPopMessage = "Texture is incomplete";
} else {
mRecentPopMessage.clear();
}
Texture* texture = item->async->getTexture();
mKtxReader->asyncDestroy(&item->async);
item->async = nullptr;
return texture;
}
}
return nullptr;
}
void Ktx2Provider::updateQueue() {
if (!UTILS_HAS_THREADING) {
transcodeSingleTexture();
}
JobSystem* js = &mEngine->getJobSystem();
for (auto& item : mQueueItems) {
if (item->state != QueueItemState::TRANSCODING) {
continue;
}
Texture* texture = item->async->getTexture();
const TranscoderState state = item->transcoderState.load();
if (state != TranscoderState::NOT_STARTED) {
if (item->job) {
js->waitAndRelease(item->job);
}
if (state == TranscoderState::ERROR) {
item->state = QueueItemState::READY;
++mDecodedCount;
continue;
}
item->async->uploadImages();
item->state = QueueItemState::READY;
++mDecodedCount;
}
}
// Here we periodically clean up the "queue" (which is really just a vector) by removing unused
// items from the front. This might ignore a popped texture that occurs in the middle of the
// vector, but that's okay, it will be cleaned up eventually.
decltype(mQueueItems)::iterator last = mQueueItems.begin();
while (last != mQueueItems.end() && (*last)->state == QueueItemState::POPPED) ++last;
mQueueItems.erase(mQueueItems.begin(), last);
}
void Ktx2Provider::waitForCompletion() {
JobSystem& js = mEngine->getJobSystem();
for (auto& item : mQueueItems) {
if (item->job) {
js.waitAndRelease(item->job);
}
}
}
void Ktx2Provider::cancelDecoding() {
// TODO: Currently, Ktx2Provider runs jobs eagerly and JobSystem does not allow cancellation of
// in-flight jobs. We should consider throttling the number of simultaneous decoder jobs, which
// would allow for actual cancellation.
waitForCompletion();
}
const char* Ktx2Provider::getPushMessage() const {
return mRecentPushMessage.empty() ? nullptr : mRecentPushMessage.c_str();
}
const char* Ktx2Provider::getPopMessage() const {
return mRecentPopMessage.empty() ? nullptr : mRecentPopMessage.c_str();
}
void Ktx2Provider::transcodeSingleTexture() {
assert_invariant(!UTILS_HAS_THREADING);
for (auto& item : mQueueItems) {
if (item->state == QueueItemState::TRANSCODING) {
using Result = ktxreader::Ktx2Reader::Result;
bool success = Result::SUCCESS == item->async->doTranscoding();
item->transcoderState.store(success ? TranscoderState::SUCCESS : TranscoderState::ERROR);
break;
}
}
}
Ktx2Provider::Ktx2Provider(Engine* engine) : mEngine(engine) {
mDecoderRootJob = mEngine->getJobSystem().createJob();
#ifdef NDEBUG
const bool quiet = true;
#else
const bool quiet = false;
#endif
mKtxReader.reset(new ktxreader::Ktx2Reader(*engine, quiet));
mKtxReader->requestFormat(Texture::InternalFormat::ETC2_EAC_SRGBA8);
mKtxReader->requestFormat(Texture::InternalFormat::DXT5_SRGBA);
mKtxReader->requestFormat(Texture::InternalFormat::DXT5_RGBA);
// Uncompressed formats are lower priority, so they get added last.
mKtxReader->requestFormat(Texture::InternalFormat::SRGB8_A8);
mKtxReader->requestFormat(Texture::InternalFormat::RGBA8);
}
Ktx2Provider::~Ktx2Provider() {
cancelDecoding();
for (auto& item : mQueueItems) {
mKtxReader->asyncDestroy(&item->async);
}
mEngine->getJobSystem().release(mDecoderRootJob);
}
TextureProvider* createKtx2Provider(Engine* engine) {
return new Ktx2Provider(engine);
}
} // namespace gltfio

3
libs/gltfio/src/ResourceLoader.cpp
View File

@@ -571,7 +571,8 @@ void ResourceLoader::asyncUpdateLoad() {
Texture* ResourceLoader::Impl::getOrCreateTexture(FFilamentAsset* asset, const TextureSlot& tb) {
const cgltf_texture* srcTexture = tb.texture;
const cgltf_image* image = srcTexture->image;
const cgltf_image* image = srcTexture->basisu_image ?
srcTexture->basisu_image : srcTexture->image;
const cgltf_buffer_view* bv = image->buffer_view;
const char* uri = image->uri;

2
libs/gltfio/src/StbProvider.cpp
View File

@@ -35,7 +35,7 @@ using std::unique_ptr;
namespace gltfio {
class StbProvider : public TextureProvider {
class StbProvider final : public TextureProvider {
public:
StbProvider(Engine* engine);
~StbProvider();

14
samples/gltf_viewer.cpp
View File

@@ -85,7 +85,8 @@ struct App {
MaterialSource materialSource = GENERATE_SHADERS;
gltfio::ResourceLoader* resourceLoader = nullptr;
gltfio::TextureProvider* decoder = nullptr;
gltfio::TextureProvider* stbDecoder = nullptr;
gltfio::TextureProvider* ktxDecoder = nullptr;
bool recomputeAabb = false;
bool ignoreBindTransform = false;
@@ -425,9 +426,11 @@ int main(int argc, char** argv) {
configuration.normalizeSkinningWeights = true;
if (!app.resourceLoader) {
app.resourceLoader = new gltfio::ResourceLoader(configuration);
app.decoder = createStbProvider(app.engine);
app.resourceLoader->addTextureProvider("image/png", app.decoder);
app.resourceLoader->addTextureProvider("image/jpeg", app.decoder);
app.stbDecoder = createStbProvider(app.engine);
app.ktxDecoder = createKtx2Provider(app.engine);
app.resourceLoader->addTextureProvider("image/png", app.stbDecoder);
app.resourceLoader->addTextureProvider("image/jpeg", app.stbDecoder);
app.resourceLoader->addTextureProvider("image/ktx2", app.ktxDecoder);
}
app.resourceLoader->asyncBeginLoad(app.asset);
app.asset->releaseSourceData();
@@ -650,7 +653,8 @@ int main(int argc, char** argv) {
delete app.viewer;
delete app.materials;
delete app.names;
delete app.decoder;
delete app.stbDecoder;
delete app.ktxDecoder;
AssetLoader::destroy(&app.assetLoader);
};

6
web/filament-js/extensions.js
View File

@@ -645,9 +645,11 @@ Filament.loadClassExtensions = function() {
config.ignoreBindTransform);
const stbProvider = new Filament.gltfio$StbProvider(engine);
const ktx2Provider = new Filament.gltfio$Ktx2Provider(engine);
resourceLoader.addTextureProvider("image/jpeg", stbProvider);
resourceLoader.addTextureProvider("image/png", stbProvider);
resourceLoader.addStbProvider("image/jpeg", stbProvider);
resourceLoader.addStbProvider("image/png", stbProvider);
resourceLoader.addKtx2Provider("image/ktx2", ktx2Provider);
const onComplete = () => {
resourceLoader.asyncBeginLoad(asset);

11
web/filament-js/filament-viewer.js
View File

@@ -309,7 +309,7 @@ class FilamentViewer extends LitElement {
asyncInterval: 30
};
const doneAddingResources = (resourceLoader, stbProvider) => {
const doneAddingResources = (resourceLoader, stbProvider, ktx2Provider) => {
this.srcBlobResources = {};
resourceLoader.asyncBeginLoad(this.asset);
const timer = setInterval(() => {
@@ -319,6 +319,7 @@ class FilamentViewer extends LitElement {
clearInterval(timer);
resourceLoader.delete();
stbProvider.delete();
ktx2Provider.delete();
this.animator = this.asset.getAnimator();
this.animationStartTime = Date.now();
}
@@ -337,9 +338,11 @@ class FilamentViewer extends LitElement {
config.ignoreBindTransform);
const stbProvider = new Filament.gltfio$StbProvider(this.engine);
const ktx2Provider = new Filament.gltfio$Ktx2Provider(this.engine);
resourceLoader.addTextureProvider("image/jpeg", stbProvider);
resourceLoader.addTextureProvider("image/png", stbProvider);
resourceLoader.addStbProvider("image/jpeg", stbProvider);
resourceLoader.addStbProvider("image/png", stbProvider);
resourceLoader.addKtx2Provider("image/ktx2", ktx2Provider);
let remaining = Object.keys(this.srcBlobResources).length;
for (const name in this.srcBlobResources) {
@@ -347,7 +350,7 @@ class FilamentViewer extends LitElement {
const desc = getBufferDescriptor(new Uint8Array(buffer));
resourceLoader.addResourceData(name, getBufferDescriptor(desc));
if (--remaining === 0) {
doneAddingResources(resourceLoader, stbProvider);
doneAddingResources(resourceLoader, stbProvider, ktx2Provider);
}
});
}

17
web/filament-js/jsbindings.cpp
View File

@@ -1895,9 +1895,8 @@ struct UbershaderLoader {
void destroyMaterials() { provider->destroyMaterials(); }
};
struct StbProvider {
TextureProvider* provider;
};
struct StbProvider { TextureProvider* provider; };
struct Ktx2Provider { TextureProvider* provider; };
class_<UbershaderLoader>("gltfio$UbershaderLoader")
.constructor(EMBIND_LAMBDA(UbershaderLoader, (Engine* engine), {
@@ -1910,6 +1909,11 @@ class_<StbProvider>("gltfio$StbProvider")
return StbProvider { createStbProvider(engine) };
}));
class_<Ktx2Provider>("gltfio$Ktx2Provider")
.constructor(EMBIND_LAMBDA(Ktx2Provider, (Engine* engine), {
return Ktx2Provider { createKtx2Provider(engine) };
}));
class_<AssetLoader>("gltfio$AssetLoader")
.constructor(EMBIND_LAMBDA(AssetLoader*, (Engine* engine, UbershaderLoader materials), {
@@ -1971,11 +1975,16 @@ class_<ResourceLoader>("gltfio$ResourceLoader")
self->addResourceData(url.c_str(), std::move(*buffer.bd));
}), allow_raw_pointers())
.function("addTextureProvider", EMBIND_LAMBDA(void, (ResourceLoader* self, std::string mime,
.function("addStbProvider", EMBIND_LAMBDA(void, (ResourceLoader* self, std::string mime,
StbProvider provider), {
self->addTextureProvider(mime.c_str(), provider.provider);
}), allow_raw_pointers())
.function("addKtx2Provider", EMBIND_LAMBDA(void, (ResourceLoader* self, std::string mime,
Ktx2Provider provider), {
self->addTextureProvider(mime.c_str(), provider.provider);
}), allow_raw_pointers())
.function("hasResourceData", EMBIND_LAMBDA(bool, (ResourceLoader* self, std::string url), {
return self->hasResourceData(url.c_str());
}), allow_raw_pointers())