From 80f6f18e54bc3d213553fc085fdbcdb3eb9bbe82 Mon Sep 17 00:00:00 2001 From: Philip Rideout Date: Thu, 9 May 2019 14:13:56 -0700 Subject: [PATCH] Introduce gltfio::PathTracer and baking API. AssetPipeline now has a path tracer and optionally uses embree. For now, the path tracer only sends out visibility rays and generates a simple hit-or-miss monochrome bitmap. It splits the render target into tiles and invokes a JobSystem task for each tile. Filament developers can optionally install embree using homebrew or a debian package. This avoids bloating third_party and increasing our build time. If embree is not installed, a friendly run-time error will occur when attempting to invoke the rendering functionality in AssetPipeline. The AO code is not yet implemented, but this PR sets up the infrastructure, API, and camera rays. --- libs/gltfio/CMakeLists.txt | 26 +- libs/gltfio/include/gltfio/AssetPipeline.h | 47 +++- libs/gltfio/include/gltfio/SimpleCamera.h | 35 +++ libs/gltfio/src/AssetPipeline.cpp | 49 +++- libs/gltfio/src/PathTracer.cpp | 281 +++++++++++++++++++++ libs/gltfio/src/PathTracer.h | 72 ++++++ 6 files changed, 499 insertions(+), 11 deletions(-) create mode 100644 libs/gltfio/include/gltfio/SimpleCamera.h create mode 100644 libs/gltfio/src/PathTracer.cpp create mode 100644 libs/gltfio/src/PathTracer.h diff --git a/libs/gltfio/CMakeLists.txt b/libs/gltfio/CMakeLists.txt index fa7e1bcb8e..9a412529ec 100644 --- a/libs/gltfio/CMakeLists.txt +++ b/libs/gltfio/CMakeLists.txt @@ -13,6 +13,7 @@ set(PUBLIC_HDRS include/gltfio/MaterialProvider.h include/gltfio/AssetPipeline.h include/gltfio/ResourceLoader.h + include/gltfio/SimpleCamera.h include/gltfio/SimpleViewer.h include/gltfio/FilamentAsset.h ) @@ -106,10 +107,22 @@ add_custom_command(OUTPUT ${DUMMY_SRC} COMMAND echo "//" > ${DUMMY_SRC}) add_library(gltfio_resources ${DUMMY_SRC} ${RESGEN_SOURCE}) +# ================================================================================================== +# Check if embree is available. +# This is an optional dependency and can be installed with homebrew, apt-get, etc. +# ================================================================================================== + +find_package(embree 3.0) +if (embree_FOUND) + message("Found embree in ${embree_DIR}") + add_definitions(-DFILAMENT_HAS_EMBREE) +endif() + # ================================================================================================== # Build the "core" library (no filamat or pipeline) # ================================================================================================== -include_directories(${PUBLIC_HDR_DIR} ${RESOURCE_DIR}) + +include_directories(${PUBLIC_HDR_DIR} ${RESOURCE_DIR} ${EMBREE_INCLUDE_DIRS}) link_libraries(math utils filament cgltf stb geometry gltfio_resources) add_library(gltfio_core STATIC ${PUBLIC_HDRS} ${SRCS}) @@ -123,8 +136,15 @@ if (NOT WEBGL AND NOT ANDROID AND NOT IOS) # ================================================================================================== add_library(${TARGET} STATIC ${PUBLIC_HDRS} src/MaterialGenerator.cpp - src/AssetPipeline.cpp) - target_link_libraries(${TARGET} PUBLIC filamat xatlas meshoptimizer gltfio_core) + src/AssetPipeline.cpp + src/PathTracer.cpp + src/PathTracer.h) + target_link_libraries(${TARGET} PUBLIC + filamat + xatlas + meshoptimizer + gltfio_core + ${EMBREE_LIBRARY}) target_include_directories(${TARGET} PUBLIC ${PUBLIC_HDR_DIR}) # ================================================================================================== diff --git a/libs/gltfio/include/gltfio/AssetPipeline.h b/libs/gltfio/include/gltfio/AssetPipeline.h index 512b1d0b4a..1f8485f1f4 100644 --- a/libs/gltfio/include/gltfio/AssetPipeline.h +++ b/libs/gltfio/include/gltfio/AssetPipeline.h @@ -17,10 +17,14 @@ #ifndef GLTFIO_ASSETPIPELINE_H #define GLTFIO_ASSETPIPELINE_H -#include +#include + +#include #include +#include + namespace gltfio { /** @@ -81,14 +85,53 @@ public: static constexpr uint32_t DISCARD_TEXTURES = 1 << 0; static constexpr uint32_t FILTER_TRIANGLES = 1 << 1; + static const char* const BAKED_UV_ATTRIB; + static const int BAKED_UV_ATTRIB_INDEX; + /** * Generates atlases and UV data for a flattened asset that are suitable for baking lightmaps. * * The topology of the meshes in the resulting asset is potentially different from the source - * asset (e.g., new vertices might be inserted). + * asset (e.g., new vertices might be inserted). The newly generated UV set is placed into the + * attribute slot named BAKED_UV_ATTRIB. */ AssetHandle parameterize(AssetHandle source); + /** + * Signals that a region of a path-traced image is available (used for progress notification). + * This can be called from any thread. + */ + using RenderTileCallback = void(*)(image::LinearImage target, + filament::math::ushort2 topLeft, filament::math::ushort2 bottomRight, + void* userData); + + /** + * Signals that a complete path-traced image has become available. + * This can be called from any thread. + */ + using RenderDoneCallback = void(*)(image::LinearImage target, void* userData); + + /** + * Consumes a parameterized glTF asset and produces a single-channel image with ambient + * occlusion. Requires the presence of BAKED_UV_ATTRIB in the source asset. + * + * This invokes a simple path tracer that operates on tiles of the target image. It spins + * up a number of threads and triggers a callback every time a new tile has been rendered. + */ + void bakeAmbientOcclusion(AssetHandle source, image::LinearImage target, + RenderTileCallback progress, RenderDoneCallback done, void* userData); + + /** + * Consumes a glTF asset and produces a single-channel image with ambient occlusion rendered + * from the given camera. + * + * This method is not related to baking and was initially authored for diagnostic purposes. + * The progress callback is similar to bakeAmbientOcclusion. + */ + void renderAmbientOcclusion(AssetHandle source, image::LinearImage target, + const SimpleCamera& camera, RenderTileCallback progress, RenderDoneCallback done, + void* userData); + AssetPipeline(); ~AssetPipeline(); diff --git a/libs/gltfio/include/gltfio/SimpleCamera.h b/libs/gltfio/include/gltfio/SimpleCamera.h new file mode 100644 index 0000000000..73695acf72 --- /dev/null +++ b/libs/gltfio/include/gltfio/SimpleCamera.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GLTFIO_SIMPLECAMERA_H +#define GLTFIO_SIMPLECAMERA_H + +#include + +namespace gltfio { + +/** Used to configure the embedded path tracer in AssetPipeline. */ +struct SimpleCamera { + float aspectRatio; + filament::math::float3 eyePosition; + filament::math::float3 targetPosition; + filament::math::float3 upVector; + float vfovDegrees; +}; + +} // namespace gltfio + +#endif // GLTFIO_SIMPLECAMERA_H diff --git a/libs/gltfio/src/AssetPipeline.cpp b/libs/gltfio/src/AssetPipeline.cpp index 2c9047f9ab..39de2dc019 100644 --- a/libs/gltfio/src/AssetPipeline.cpp +++ b/libs/gltfio/src/AssetPipeline.cpp @@ -21,6 +21,8 @@ #define CGLTF_WRITE_IMPLEMENTATION #include "cgltf_write.h" +#include "PathTracer.h" + #include #include @@ -35,6 +37,9 @@ using namespace utils; using std::vector; +const char* const gltfio::AssetPipeline::BAKED_UV_ATTRIB = "TEXCOORD_4"; +const int gltfio::AssetPipeline::BAKED_UV_ATTRIB_INDEX = 4; + namespace { using AssetHandle = gltfio::AssetPipeline::AssetHandle; @@ -44,9 +49,6 @@ static const char* const NORMAL = "NORMAL"; static const char* const TANGENT = "TANGENT"; static const char* const GENERATOR_ID = "gltfio"; -static const char* const BAKE_UV_ATTRIB_NAME = "TEXCOORD_4"; -static const cgltf_int BAKE_UV_ATTRIB_INDEX = 4; - // Bookkeeping structure for baking a single primitive + node pair. struct BakedPrim { const cgltf_node* sourceNode; @@ -138,7 +140,7 @@ cgltf_size getNumFloats(cgltf_type type) { // amenable to subsequent pipeline operations like baking and exporting. bool isFlattened(const cgltf_data* asset) { return asset && asset->buffers_count == 1 && asset->nodes_count == asset->meshes_count && - asset->asset.generator == GENERATOR_ID; + !strcmp(asset->asset.generator, GENERATOR_ID); } // Returns true if the given primitive should be baked out, false if it should be culled away. @@ -986,9 +988,9 @@ cgltf_data* Pipeline::xatlasToCgltf(const cgltf_data* sourceAsset, const xatlas: // Create the new attribute for the baked UV's and point it to its corresponding accessor. *resultAttribute++ = { - .name = (char*) BAKE_UV_ATTRIB_NAME, + .name = (char*) gltfio::AssetPipeline::BAKED_UV_ATTRIB, .type = cgltf_attribute_type_texcoord, - .index = BAKE_UV_ATTRIB_INDEX, + .index = gltfio::AssetPipeline::BAKED_UV_ATTRIB_INDEX, .data = resultAccessor }; *resultAccessor++ = { @@ -1158,4 +1160,39 @@ AssetHandle AssetPipeline::parameterize(AssetHandle source) { return impl->parameterize((const cgltf_data*) source); } +void AssetPipeline::bakeAmbientOcclusion(AssetHandle source, image::LinearImage target, + RenderTileCallback onTile, RenderDoneCallback onDone, void* userData) { + auto sourceAsset = (const cgltf_data*) source; + if (!isFlattened(sourceAsset)) { + utils::slog.e << "Only flattened assets can be baked." << utils::io::endl; + return; + } + PathTracer pathtracer = PathTracer::Builder() + .renderTarget(target) + .uvCamera(BAKED_UV_ATTRIB) + .tileCallback(onTile, userData) + .doneCallback(onDone, userData) + .sourceAsset((cgltf_data*) source) + .build(); + pathtracer.render(); +} + +void AssetPipeline::renderAmbientOcclusion(AssetHandle source, image::LinearImage target, + const SimpleCamera& camera, RenderTileCallback onTile, + RenderDoneCallback onDone, void* userData) { + auto sourceAsset = (const cgltf_data*) source; + if (!isFlattened(sourceAsset)) { + utils::slog.e << "Only flattened assets can be rendered." << utils::io::endl; + return; + } + PathTracer pathtracer = PathTracer::Builder() + .renderTarget(target) + .filmCamera(camera) + .tileCallback(onTile, userData) + .doneCallback(onDone, userData) + .sourceAsset((cgltf_data*) source) + .build(); + pathtracer.render(); +} + } // namespace gltfio diff --git a/libs/gltfio/src/PathTracer.cpp b/libs/gltfio/src/PathTracer.cpp new file mode 100644 index 0000000000..3ebf1f300c --- /dev/null +++ b/libs/gltfio/src/PathTracer.cpp @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PathTracer.h" + +#include +#include + +#include + +#include + +#ifdef FILAMENT_HAS_EMBREE +#include +#include +#endif + +using namespace filament::math; + +static constexpr uint16_t TILE_SIZE = 32; +static constexpr float inf = std::numeric_limits::infinity(); + +struct PixelRectangle { + ushort2 topLeft; + ushort2 bottomRight; +}; + +static float3 sampleSphere() { + float z = 2.0f * rand() / RAND_MAX - 1.0f; + float t = 2.0f * rand() / RAND_MAX * M_PI; + float r = std::sqrt(1.0f - z * z); + return { r * std::cos(t), r * std::sin(t), z}; +} + +namespace gltfio { + +PathTracer::Builder::Builder() { + mPathTracer = new PathTracer(); +} + +PathTracer::Builder::~Builder() { + delete mPathTracer; +} + +PathTracer::Builder& PathTracer::Builder::renderTarget(image::LinearImage target) { + mPathTracer->mRenderTarget = target; + return *this; + +} + +PathTracer::Builder& PathTracer::Builder::sourceAsset(AssetHandle asset) { + mPathTracer->mSourceAsset = asset; + return *this; + +} + +PathTracer::Builder& PathTracer::Builder::filmCamera(const SimpleCamera& filmCamera) { + mPathTracer->mFilmCamera = filmCamera; + return *this; + +} + +PathTracer::Builder& PathTracer::Builder::uvCamera(const char* uvAttribute) { + mPathTracer->mUvCamera = uvAttribute; + return *this; + +} + +PathTracer::Builder& PathTracer::Builder::tileCallback(TileCallback onTile, void* userData) { + mPathTracer->mTileCallback = onTile; + mPathTracer->mTileUserData = userData; + return *this; +} + +PathTracer::Builder& PathTracer::Builder::doneCallback(DoneCallback onDone, void* userData) { + mPathTracer->mDoneCallback = onDone; + mPathTracer->mDoneUserData = userData; + return *this; +} + +PathTracer PathTracer::Builder::build() { + return *mPathTracer; +} + +#ifndef FILAMENT_HAS_EMBREE + +bool PathTracer::render() { + puts("Embree is not available."); + return false; +} + +#else + +// Note that the embree context can outlive the user-facing PathTracer object due to work +// occuring on multiple background threads. +struct EmbreeContext { + image::LinearImage renderTarget; + const cgltf_data* sourceAsset; + SimpleCamera filmCamera; + const char* uvCamera; + PathTracer::TileCallback tileCallback; + void* tileUserData; + PathTracer::DoneCallback doneCallback; + void* doneUserData; + std::atomic numRemainingTiles; + RTCDevice embreeDevice; + RTCScene embreeScene; +}; + +static void renderTile(EmbreeContext* context, PixelRectangle rect) { + image::LinearImage& image = context->renderTarget; + + // Precompute some camera parameters. + const SimpleCamera& camera = context->filmCamera; + const float iw = 1.0f / image.getWidth(); + const float ih = 1.0f / image.getHeight(); + const float theta = camera.vfovDegrees * M_PI / 180; + const float f = tanf(theta / 2); + const float a = camera.aspectRatio; + const float3 org = camera.eyePosition; + const uint16_t hm1 = image.getHeight() - 1; + + // Compute the camera basis: view, right, and up vectors. + const float3 v = normalize(camera.targetPosition - org); + const float3 r = normalize(cross(v, camera.upVector)); + const float3 u = cross(r, v); + + // Given a pixel row and column, generate a ray from the eye through the film. + auto generateCameraRay = [=] (uint16_t row, uint16_t col) { + const uint16_t x = col; + const uint16_t y = hm1 - row; + const float s = (2.0f * (x + 0.5f) * iw - 1.0f); + const float t = (2.0f * (y + 0.5f) * ih - 1.0f); + const float3 dir = normalize(a * f * s * r - f * t * u + v); + return RTCRay { + .org_x = org.x, + .org_y = org.y, + .org_z = org.z, + .tnear = 0, + .dir_x = dir.x, + .dir_y = dir.y, + .dir_z = dir.z, + .time = 0, + .tfar = inf, + .mask = 0xffffffff + }; + }; + + // Loop over all pixels in the tile. + for (size_t row = rect.topLeft.y, len = rect.bottomRight.y; row < len; ++row) { + for (size_t col = rect.topLeft.x, len = rect.bottomRight.x; col < len; ++col) { + RTCIntersectContext intersector; + rtcInitIntersectContext(&intersector); + RTCRay ray = generateCameraRay(row, col); + + // For now perform a simple visibility test and set the pixel to white or black. + rtcOccluded1(context->embreeScene, &intersector, &ray); + float* dst = image.getPixelRef(col, row); + *dst = (ray.tfar == -inf) ? 1.0f : 0.0f; + } + } + + // Signal that we're done with the tile. For simplicity we decrement an atomic tile count + // to know when we're done with the entire image. + context->tileCallback(image, rect.topLeft, rect.bottomRight, context->tileUserData); + if (--context->numRemainingTiles == 0) { + context->doneCallback(image, context->doneUserData); + rtcReleaseScene(context->embreeScene); + rtcReleaseDevice(context->embreeDevice); + delete context; + } +} + +bool PathTracer::render() { + auto sourceAsset = (const cgltf_data*) mSourceAsset; + EmbreeContext* context = new EmbreeContext { + .renderTarget = mRenderTarget, + .sourceAsset = sourceAsset, + .filmCamera = mFilmCamera, + .uvCamera = mUvCamera, + .tileCallback = mTileCallback, + .tileUserData = mTileUserData, + .doneCallback = mDoneCallback, + .doneUserData = mDoneUserData + }; + + // Create the embree device and scene. + auto device = context->embreeDevice = rtcNewDevice(nullptr); + assert(device && "Unable to create embree device."); + auto scene = context->embreeScene = rtcNewScene(device); + assert(scene); + + // Create an embree Geometry object. For AO rendering we only care about positions and indices. + auto createGeometry = [device](const cgltf_accessor* positions, const cgltf_accessor* indices) { + assert(positions->component_type == cgltf_component_type_r_32f); + assert(positions->type == cgltf_type_vec3); + assert(indices->component_type == cgltf_component_type_r_32u); + assert(indices->type == cgltf_type_scalar); + + auto geo = rtcNewGeometry(device, RTC_GEOMETRY_TYPE_TRIANGLE); + + rtcSetSharedGeometryBuffer(geo, RTC_BUFFER_TYPE_VERTEX, 0, RTC_FORMAT_FLOAT3, + positions->buffer_view->buffer->data, + positions->offset + positions->buffer_view->offset, + positions->stride, + positions->count); + + rtcSetSharedGeometryBuffer(geo, RTC_BUFFER_TYPE_INDEX, 0, RTC_FORMAT_UINT3, + indices->buffer_view->buffer->data, + indices->offset + indices->buffer_view->offset, + indices->stride * 3, + indices->count / 3); + + rtcCommitGeometry(geo); + return geo; + }; + + // Populate the embree mesh from a flattened glTF asset, which is guaranteed to have only + // one mesh per node, and only one primitive per mesh. + cgltf_node** nodes = sourceAsset->scene->nodes; + for (cgltf_size i = 0, len = sourceAsset->scene->nodes_count; i < len; ++i) { + assert(nodes[i]->mesh); + assert(nodes[i]->mesh->primitives_count == 1); + const cgltf_primitive& prim = nodes[i]->mesh->primitives[0]; + for (cgltf_size j = 0, len = prim.attributes_count; j < len; ++j) { + const cgltf_attribute& attr = prim.attributes[j]; + if (attr.type == cgltf_attribute_type_position) { + auto geo = createGeometry(attr.data, prim.indices); + rtcAttachGeometry(scene, geo); + rtcReleaseGeometry(geo); + } + } + } + rtcCommitScene(scene); + + // Compute the number of jobs by pre-running the loop. + const size_t width = mRenderTarget.getWidth(); + const size_t height = mRenderTarget.getHeight(); + int numTiles = 0; + for (size_t row = 0; row < height; row += TILE_SIZE) { + for (size_t col = 0; col < width; col += TILE_SIZE, ++numTiles); + } + context->numRemainingTiles = numTiles; + + // Kick off one job per tile. + utils::JobSystem* js = utils::JobSystem::getJobSystem(); + utils::JobSystem::Job* parent = js->createJob(); + for (size_t row = 0; row < height; row += TILE_SIZE) { + for (size_t col = 0; col < width; col += TILE_SIZE) { + PixelRectangle rect; + rect.topLeft = {col, row}; + rect.bottomRight = {col + TILE_SIZE, row + TILE_SIZE}; + rect.bottomRight.x = std::min(rect.bottomRight.x, (uint16_t) width); + rect.bottomRight.y = std::min(rect.bottomRight.y, (uint16_t) height); + utils::JobSystem::Job* tile = utils::jobs::createJob(*js, parent, [context, rect] { + renderTile(context, rect); + }); + js->run(tile); + } + } + js->run(parent); + + return true; +} + +#endif // FILAMENT_HAS_EMBREE + +} // namespace gltfio diff --git a/libs/gltfio/src/PathTracer.h b/libs/gltfio/src/PathTracer.h new file mode 100644 index 0000000000..20b98d4bc9 --- /dev/null +++ b/libs/gltfio/src/PathTracer.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GLTFIO_PATHTRACER_H +#define GLTFIO_PATHTRACER_H + +#include +#include + +#include + +namespace gltfio { + +/** + * PathTracer uses Intel embree to render an AssetPipeline scene by splitting the render target + * into tiles and invoking a JobSystem task for each tile. + * + * The lifetime of the PathTracer object is independent of the background rendering process so + * clients can create a PathTracer and immediately discard it. The passed-in progress callbacks are + * triggered from a separate thread. + */ +class PathTracer { +public: + using TileCallback = AssetPipeline::RenderTileCallback; + using DoneCallback = AssetPipeline::RenderDoneCallback; + using AssetHandle = AssetPipeline::AssetHandle; + + class Builder { + public: + Builder(); + ~Builder(); + Builder& renderTarget(image::LinearImage target); + Builder& sourceAsset(AssetHandle asset); + Builder& filmCamera(const SimpleCamera& camera); + Builder& uvCamera(const char* uvAttribute); + Builder& tileCallback(TileCallback onTile, void* userData); + Builder& doneCallback(DoneCallback onTile, void* userData); + PathTracer build(); + private: + PathTracer* mPathTracer; + }; + + bool render(); + +private: + PathTracer() {} + image::LinearImage mRenderTarget; + AssetHandle mSourceAsset; + SimpleCamera mFilmCamera; + const char* mUvCamera = nullptr; + TileCallback mTileCallback = nullptr; + void* mTileUserData = nullptr; + DoneCallback mDoneCallback = nullptr; + void* mDoneUserData = nullptr; +}; + +} // namespace gltfio + +#endif // GLTFIO_PATHTRACER_H