Compare commits

..

1 Commits

Author SHA1 Message Date
Powei Feng
f2ae43e0e1 Command buffer overflow repro 2026-02-16 10:30:53 -08:00
12 changed files with 124 additions and 413 deletions

View File

@@ -67,16 +67,7 @@ jobs:
# Only build 1 64 bit target during presubmit to cut down build times during presubmit
# Continuous builds will build everything
run: |
pushd .
cd build/android && printf "y" | ./build.sh presubmit-with-archive arm64-v8a
popd
- name: Check artifact sizes
run: |
python3 test/sizeguard/dump_artifact_size.py out/*.aar > current_size.json
python3 test/sizeguard/check_size.py current_size.json \
--target-branch origin/main \
--threshold 20480 \
--artifacts filament-android-release.aar/jni/arm64-v8a/libfilament-jni.so
cd build/android && printf "y" | ./build.sh presubmit arm64-v8a
build-ios:
name: build-iOS

View File

@@ -39,11 +39,6 @@ if [[ "$TARGET" == "presubmit-with-test" ]]; then
RUN_TESTS=-u
fi
if [[ "$TARGET" == "presubmit-with-archive" ]]; then
BUILD_RELEASE=release
GENERATE_ARCHIVES=-a
fi
if [[ "$TARGET" == "debug" ]]; then
BUILD_DEBUG=debug
GENERATE_ARCHIVES=-a

View File

@@ -83,22 +83,6 @@ protected:
*/
Driver* createDriver(void* sharedContext, const DriverConfig& driverConfig) override;
/**
* The implementation of *createDriver*.
* @param sharedContext an optional shared context. This is not meaningful with all graphic
* APIs and platforms.
* For EGL platforms, this is an EGLContext.
*
* @param driverConfig specifies driver initialization parameters
*
* @param initFirstbyquery determines the order of initialization. If true, then we'd query by
* eglQueryDevicesEXT first instead of using the default display. Useful
* for headless egl initialization.
* @return nullptr on failure, or a pointer to the newly created driver.
*/
Driver* createDriverBase(void* sharedContext, const DriverConfig& driverConfig,
bool initFirstByQuery);
/**
* This returns zero. This method can be overridden to return something more useful.
* @return zero

View File

@@ -19,112 +19,21 @@
#include <backend/PixelBufferDescriptor.h>
#include <math/scalar.h>
#include <math/half.h>
#include <utils/debug.h>
#include <utils/Logger.h>
#include <cstdint>
#include <cstring>
#include <stddef.h>
#include <stdint.h>
#include <math/scalar.h>
#include <utils/debug.h>
namespace filament {
namespace backend {
namespace {
// Provides an alpha value when expanding 3-channel images to 4-channel.
// Also used as a normalization scale when converting between numeric types.
template<typename componentType> inline componentType getMaxValue();
template<> inline constexpr float getMaxValue() { return 1.0f; }
template<> inline constexpr int32_t getMaxValue() { return 0x7fffffff; }
template<> inline constexpr uint32_t getMaxValue() { return 0xffffffff; }
template<> inline constexpr uint16_t getMaxValue() { return 0x3c00; } // 0x3c00 is 1.0 in half-float.
template<> inline constexpr uint8_t getMaxValue() { return 0xff; }
template<> inline math::half getMaxValue() { return math::half(1.0f); }
// We use template below to reduce code duplication across the different input/output
// type/channle-count permutations. Morever, templates help us reduce the number of conditionals
// in the inner-loop of the reshape operation. However, this needs to be a carefully considered
// because too many templated params will cause a large binary size increase.
// Note that we intentionally do not want to expand the template params to include the channel count
// because of the size increase.
template<typename dstComponentType, bool hasAlpha>
void grayscaleFill(dstComponentType* dst, uint8_t, uint8_t) {
for (size_t channel = 1; channel < 3; ++channel) {
dst[channel] = dst[0];
}
if constexpr (hasAlpha) {
dst[3] = getMaxValue<dstComponentType>();
}
}
// Note that we intentionally do not want to expand the template params to include the channel count
// because of the size increase.
template<typename dstComponentType>
inline void maxValFill(dstComponentType* dst, uint8_t srcChannelCount, uint8_t dstChannelCount) {
dstComponentType dstMaxValue = getMaxValue<dstComponentType>();
for (size_t channel = srcChannelCount; channel < dstChannelCount; ++channel) {
dst[channel] = dstMaxValue;
}
}
// Converts a n-channel image of UBYTE, INT, UINT, HALF, or FLOAT to a different type.
template<typename dstComponentType, typename srcComponentType>
void reshapeImageImpl(uint8_t* UTILS_RESTRICT dest, const uint8_t* UTILS_RESTRICT src,
size_t srcBytesPerRow, size_t srcChannelCount, size_t dstRowOffset, size_t dstColumnOffset,
size_t dstBytesPerRow, size_t dstChannelCount, size_t width, size_t height, bool swizzle) {
static_assert(!std::is_same_v<dstComponentType, math::half>);
const size_t minChannelCount = math::min(srcChannelCount, dstChannelCount);
const dstComponentType dstMaxValue = getMaxValue<dstComponentType>();
const srcComponentType srcMaxValue = getMaxValue<srcComponentType>();
double const mFactor = dstMaxValue / ((double) srcMaxValue);
assert_invariant(minChannelCount <= 4);
UTILS_ASSUME(minChannelCount <= 4);
dest += (dstRowOffset * dstBytesPerRow);
void (*fill)(dstComponentType*, uint8_t, uint8_t);
if (srcChannelCount == 1 && dstChannelCount == 3) {
fill = grayscaleFill<dstComponentType, false>;
} else if (srcChannelCount == 1 && dstChannelCount == 4) {
fill = grayscaleFill<dstComponentType, true>;
} else {
fill = maxValFill<dstComponentType>;
}
const int inds[4] = { swizzle ? 2 : 0, 1, swizzle ? 0 : 2, 3 };
for (size_t row = 0; row < height; ++row) {
const srcComponentType* in = (const srcComponentType*) src;
dstComponentType* out = (dstComponentType*) dest + (dstColumnOffset * dstChannelCount);
for (size_t column = 0; column < width; ++column) {
for (uint8_t channel = 0; channel < minChannelCount; ++channel) {
if constexpr (std::is_same_v<dstComponentType, srcComponentType>) {
out[channel] = in[inds[channel]];
} else {
// convert to double then clamp and cast to dst type.
out[channel] = static_cast<dstComponentType>(std::clamp(
in[inds[channel]] * mFactor, 0.0,
static_cast<double>(std::numeric_limits<dstComponentType>::max())));
}
}
// This will fill in all the channels that are not copied.
fill(out, srcChannelCount, dstChannelCount);
in += srcChannelCount;
out += dstChannelCount;
}
src += srcBytesPerRow;
dest += dstBytesPerRow;
}
}
} // anonymous namespace
class DataReshaper {
public:
@@ -167,9 +76,51 @@ public:
}
}
// Converts a n-channel image of UBYTE, INT, UINT, or FLOAT to a different type.
template<typename dstComponentType, typename srcComponentType>
static void reshapeImage(uint8_t* UTILS_RESTRICT dest, const uint8_t* UTILS_RESTRICT src,
size_t srcBytesPerRow,
size_t srcChannelCount,
size_t dstRowOffset, size_t dstColumnOffset,
size_t dstBytesPerRow, size_t dstChannelCount,
size_t width, size_t height, bool swizzle) {
// TODO: there's a fast-path where memcpy will work but currently not being taken advantage
// of.
const dstComponentType dstMaxValue = getMaxValue<dstComponentType>();
const srcComponentType srcMaxValue = getMaxValue<srcComponentType>();
const size_t minChannelCount = math::min(srcChannelCount, dstChannelCount);
assert_invariant(minChannelCount <= 4);
UTILS_ASSUME(minChannelCount <= 4);
dest += (dstRowOffset * dstBytesPerRow);
const int inds[4] = { swizzle ? 2 : 0, 1, swizzle ? 0 : 2, 3 };
for (size_t row = 0; row < height; ++row) {
const srcComponentType* in = (const srcComponentType*) src;
dstComponentType* out = (dstComponentType*)dest + (dstColumnOffset * dstChannelCount);
for (size_t column = 0; column < width; ++column) {
for (size_t channel = 0; channel < minChannelCount; ++channel) {
if constexpr (std::is_same_v<dstComponentType, srcComponentType>) {
out[channel] = in[inds[channel]];
} else {
// FIXME: beware of overflows in the multiply
// FIXME: probably not correct for _INTEGER src/dst
out[channel] = in[inds[channel]] * dstMaxValue / srcMaxValue;
}
}
for (size_t channel = srcChannelCount; channel < dstChannelCount; ++channel) {
out[channel] = dstMaxValue;
}
in += srcChannelCount;
out += dstChannelCount;
}
src += srcBytesPerRow;
dest += dstBytesPerRow;
}
}
// Converts a n-channel image of UBYTE, INT, UINT, or FLOAT to a different type.
static bool reshapeImage(PixelBufferDescriptor* UTILS_RESTRICT dst, PixelDataType srcType,
uint32_t srcChannelCount, const uint8_t* UTILS_RESTRICT srcBytes, int srcBytesPerRow,
uint32_t srcChannelCount, const uint8_t* UTILS_RESTRICT srcBytes, int srcBytesPerRow,
int width, int height, bool swizzle) {
size_t dstChannelCount;
switch (dst->format) {
@@ -181,14 +132,13 @@ public:
case PixelDataFormat::RG: dstChannelCount = 2; break;
case PixelDataFormat::RGB: dstChannelCount = 3; break;
case PixelDataFormat::RGBA: dstChannelCount = 4; break;
default:
LOG(ERROR) << "DataReshaper: unsupported dst->format: " << (int) dst->format;
return false;
default: return false;
}
void (*reshaper)(uint8_t* dest, const uint8_t* src, size_t srcBytesPerRow,
size_t srcChannelCount, size_t srcRowOffset, size_t srcColumnOffset,
size_t dstBytesPerRow, size_t dstChannelCount, size_t width, size_t height,
bool swizzle) = nullptr;
size_t srcChannelCount,
size_t srcRowOffset, size_t srcColumnOffset,
size_t dstBytesPerRow, size_t dstChannelCount,
size_t width, size_t height, bool swizzle) = nullptr;
constexpr auto UBYTE = PixelDataType::UBYTE;
constexpr auto FLOAT = PixelDataType::FLOAT;
constexpr auto UINT = PixelDataType::UINT;
@@ -198,84 +148,71 @@ public:
case UBYTE:
switch (srcType) {
case UBYTE:
reshaper = reshapeImageImpl<uint8_t, uint8_t>;
reshaper = reshapeImage<uint8_t, uint8_t>;
if (dst->format == PixelDataFormat::RGBA &&
dstChannelCount == srcChannelCount && !swizzle && dst->top == 0 &&
dst->left == 0) {
reshaper = copyImage;
}
break;
case FLOAT: reshaper = reshapeImageImpl<uint8_t, float>; break;
case INT: reshaper = reshapeImageImpl<uint8_t, int32_t>; break;
case UINT: reshaper = reshapeImageImpl<uint8_t, uint32_t>; break;
case HALF: reshaper = reshapeImageImpl<uint8_t, math::half>; break;
default:
LOG(ERROR) << "DataReshaper: UBYTE dst, unsupported srcType: "
<< (int) srcType;
return false;
case FLOAT: reshaper = reshapeImage<uint8_t, float>; break;
case INT: reshaper = reshapeImage<uint8_t, int32_t>; break;
case UINT: reshaper = reshapeImage<uint8_t, uint32_t>; break;
default: return false;
}
break;
case FLOAT:
switch (srcType) {
case UBYTE: reshaper = reshapeImageImpl<float, uint8_t>; break;
case FLOAT: reshaper = reshapeImageImpl<float, float>; break;
case INT: reshaper = reshapeImageImpl<float, int32_t>; break;
case UINT: reshaper = reshapeImageImpl<float, uint32_t>; break;
default:
LOG(ERROR) << "DataReshaper: FLOAT dst, unsupported srcType: "
<< (int) srcType;
return false;
case UBYTE: reshaper = reshapeImage<float, uint8_t>; break;
case FLOAT: reshaper = reshapeImage<float, float>; break;
case INT: reshaper = reshapeImage<float, int32_t>; break;
case UINT: reshaper = reshapeImage<float, uint32_t>; break;
default: return false;
}
break;
case INT:
switch (srcType) {
case UBYTE: reshaper = reshapeImageImpl<int32_t, uint8_t>; break;
case FLOAT: reshaper = reshapeImageImpl<int32_t, float>; break;
case INT: reshaper = reshapeImageImpl<int32_t, int32_t>; break;
case UINT: reshaper = reshapeImageImpl<int32_t, uint32_t>; break;
default:
LOG(ERROR)
<< "DataReshaper: INT dst, unsupported srcType: " << (int) srcType;
return false;
case UBYTE: reshaper = reshapeImage<int32_t, uint8_t>; break;
case FLOAT: reshaper = reshapeImage<int32_t, float>; break;
case INT: reshaper = reshapeImage<int32_t, int32_t>; break;
case UINT: reshaper = reshapeImage<int32_t, uint32_t>; break;
default: return false;
}
break;
case UINT:
switch (srcType) {
case UBYTE: reshaper = reshapeImageImpl<uint32_t, uint8_t>; break;
case FLOAT: reshaper = reshapeImageImpl<uint32_t, float>; break;
case INT: reshaper = reshapeImageImpl<uint32_t, int32_t>; break;
case UINT: reshaper = reshapeImageImpl<uint32_t, uint32_t>; break;
default:
LOG(ERROR)
<< "DataReshaper: UINT dst, unsupported srcType: " << (int) srcType;
return false;
case UBYTE: reshaper = reshapeImage<uint32_t, uint8_t>; break;
case FLOAT: reshaper = reshapeImage<uint32_t, float>; break;
case INT: reshaper = reshapeImage<uint32_t, int32_t>; break;
case UINT: reshaper = reshapeImage<uint32_t, uint32_t>; break;
default: return false;
}
break;
case HALF:
switch (srcType) {
case HALF:
reshaper = copyImage;
break;
default:
LOG(ERROR)
<< "DataReshaper: HALF dst, unsupported srcType: " << (int) srcType;
return false;
case HALF: reshaper = copyImage; break;
default: return false;
}
break;
default:
LOG(ERROR) << "DataReshaper: unsupported dst->type: " << (int) dst->type;
return false;
}
uint8_t* dstBytes = (uint8_t*) dst->buffer;
const int dstBytesPerRow = PixelBufferDescriptor::computeDataSize(dst->format, dst->type,
dst->stride ? dst->stride : width, 1, dst->alignment);
reshaper(dstBytes, srcBytes, srcBytesPerRow, srcChannelCount, dst->top, dst->left,
dstBytesPerRow, dstChannelCount, width, height, swizzle);
reshaper(dstBytes, srcBytes, srcBytesPerRow, srcChannelCount,
dst->top, dst->left, dstBytesPerRow,
dstChannelCount, width, height, swizzle);
return true;
}
};
template<> inline float getMaxValue() { return 1.0f; }
template<> inline int32_t getMaxValue() { return 0x7fffffff; }
template<> inline uint32_t getMaxValue() { return 0xffffffff; }
template<> inline uint16_t getMaxValue() { return 0x3c00; } // 0x3c00 is 1.0 in half-float.
template<> inline uint8_t getMaxValue() { return 0xff; }
} // namespace backend
} // namespace filament

View File

@@ -120,28 +120,17 @@ bool PlatformEGL::isOpenGL() const noexcept {
PlatformEGL::ExternalImageEGL::~ExternalImageEGL() = default;
Driver* PlatformEGL::createDriver(void* sharedContext, const DriverConfig& driverConfig) {
return createDriverBase(sharedContext, driverConfig, false /* initFirstByQuery */);
}
Driver* PlatformEGL::createDriverBase(void* sharedContext, const DriverConfig& driverConfig,
bool initFirstByQuery) {
static constexpr int kMaxNumEGLDevices = 32;
EGLint major, minor;
EGLBoolean initialized = false;
using InitFunc = std::function<void()>;
InitFunc queryInit = [&]() {
PFNEGLQUERYDEVICESEXTPROC const eglQueryDevicesEXT =
PFNEGLQUERYDEVICESEXTPROC(eglGetProcAddress("eglQueryDevicesEXT"));
PFNEGLGETPLATFORMDISPLAYEXTPROC const getPlatformDisplay =
PFNEGLGETPLATFORMDISPLAYEXTPROC(eglGetProcAddress("eglGetPlatformDisplay"));
if (!eglQueryDevicesEXT || !getPlatformDisplay) {
return;
}
PFNEGLQUERYDEVICESEXTPROC const eglQueryDevicesEXT =
PFNEGLQUERYDEVICESEXTPROC(eglGetProcAddress("eglQueryDevicesEXT"));
PFNEGLGETPLATFORMDISPLAYEXTPROC const getPlatformDisplay =
PFNEGLGETPLATFORMDISPLAYEXTPROC(eglGetProcAddress("eglGetPlatformDisplay"));
if (eglQueryDevicesEXT != nullptr && getPlatformDisplay != nullptr) {
EGLint numDevices = 0;
EGLDeviceEXT eglDevices[kMaxNumEGLDevices];
if (eglQueryDevicesEXT(kMaxNumEGLDevices, eglDevices, &numDevices)) {
@@ -150,27 +139,12 @@ Driver* PlatformEGL::createDriverBase(void* sharedContext, const DriverConfig& d
initialized = eglInitialize(mEGLDisplay, &major, &minor);
}
}
};
InitFunc defaultInit = [&]() {
mEGLDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
initialized = eglInitialize(mEGLDisplay, &major, &minor);
};
// The order by which we check for display/device does matter because certain platforms have
// multiple displays/devices. We either return the first queried (and successfully init'd
// display) or just use the default display. Deciding which init path should go first is
// determined by the bool *initFirstByQuery*..
std::array<InitFunc, 2> initFuncs{ defaultInit, queryInit };
if (initFirstByQuery) {
std::swap(initFuncs[0], initFuncs[1]);
}
for (auto& initFunc: initFuncs) {
if (initialized) {
break;
}
initFunc();
if (!initialized) {
mEGLDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
assert_invariant(mEGLDisplay != EGL_NO_DISPLAY);
initialized = eglInitialize(mEGLDisplay, &major, &minor);
}
if (UTILS_UNLIKELY(!initialized)) {

View File

@@ -66,7 +66,7 @@ backend::Driver* PlatformEGLHeadless::createDriver(void* sharedContext,
return nullptr;
}
return PlatformEGL::createDriverBase(sharedContext, driverConfig, true /* initFirstByQuery */);
return PlatformEGL::createDriver(sharedContext, driverConfig);
}
} // namespace filament

View File

@@ -368,7 +368,6 @@ VulkanTexture::VulkanTexture(VkDevice device, VkPhysicalDevice physicalDevice,
bool const isProtected = any(tusage & TextureUsage::PROTECTED);
VkImageCreateInfo imageInfo{
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
.flags = isProtected ? VK_IMAGE_CREATE_PROTECTED_BIT : 0u,
.imageType = target == SamplerType::SAMPLER_3D ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D,
.format = vkFormat,
.extent = {w, h, depth},

View File

@@ -378,15 +378,10 @@ VulkanPlatform::ImageData VulkanPlatformAndroid::createVkImageFromExternal(
externalCreateInfo.pNext = &imageFormatListInfo;
}
VkImageCreateFlags imageFlags =
(isFormatSrgb(metadata.format) ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0u) |
(any(metadata.filamentUsage & TextureUsage::PROTECTED)
? VK_IMAGE_CREATE_PROTECTED_BIT
: 0u);
VkImageCreateInfo const imageInfo = {
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
.pNext = &externalCreateInfo,
.flags = imageFlags,
.flags = isFormatSrgb(metadata.format) ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0u,
.imageType = VK_IMAGE_TYPE_2D,
// For non external images, use the same format as the AHB, which isn't in SRGB
// Fix VUID-VkMemoryAllocateInfo-pNext-02387

View File

@@ -901,6 +901,10 @@ void RenderPass::Executor::execute(FEngine const& engine, DriverApi& driver,
size_t const capacity = engine.getMinCommandBufferSize();
CircularBuffer const& circularBuffer = driver.getCircularBuffer();
utils::slog.e <<"circularBuffer: " << circularBuffer.size() << " used: " <<
circularBuffer.getUsed() <<
" commandCount: " << last - first << utils::io::endl;
// b/479079631: Log the number of commands in this render pass.
size_t const commandCount = last - first;
if (Platform* platform = engine.getPlatform(); platform->hasDebugUpdateStatFunc()) {

View File

@@ -264,8 +264,9 @@ FEngine::FEngine(Builder const& builder) :
mLightManager(*this),
mCameraManager(*this),
mCommandBufferQueue(
builder->mConfig.minCommandBufferSizeMB * MiB,
builder->mConfig.minCommandBufferSizeMB * MiB,
builder->mConfig.commandBufferSizeMB * MiB,
// builder->mConfig.commandBufferSizeMB * MiB,
builder->mPaused),
mPerRenderPassArena(
"FEngine::mPerRenderPassAllocator",
@@ -277,6 +278,7 @@ FEngine::FEngine(Builder const& builder) :
mMainThreadId(ThreadUtils::getThreadId()),
mConfig(builder->mConfig)
{
// update all the features flags specified in the builder
for (auto const& [feature, value] : builder->mFeatureFlags) {
auto* const p = getFeatureFlagPtr(feature.c_str_safe(), true);
@@ -348,7 +350,6 @@ void FEngine::init() {
LOG(INFO) << "Backend feature level: " << int(driverApi.getFeatureLevel());
LOG(INFO) << "FEngine feature level: " << int(mActiveFeatureLevel);
mResourceAllocatorDisposer = std::make_shared<TextureCacheDisposer>(driverApi);
mFullScreenTriangleVb = downcast(VertexBuffer::Builder()
@@ -744,12 +745,13 @@ void FEngine::prepare(DriverApi& driver) {
if (item->getMaterial()->getMaterialDomain() == MaterialDomain::SURFACE) {
// If the remaining space is less than half the capacity, we flush right
// away to allow some headroom for commands that might come later.
if (UTILS_UNLIKELY(driver.getCircularBuffer().getUsed() > capacity / 2)) {
if (UTILS_UNLIKELY(driver.getCircularBuffer().getUsed() > capacity / 2) && false) {
flush();
}
item->commit(driver, uboManager);
}
});
}
if (useUboBatching) {

View File

@@ -36,13 +36,20 @@
#include <iostream>
#include <string>// for printing usage/help
#include "filament/MaterialInstance.h"
#include "generated/resources/resources.h"
#include "generated/resources/monkey.h"
#include <utils/Log.h>
using namespace filament;
using namespace filamesh;
using namespace filament::math;
namespace {
std::vector<MaterialInstance*> instances;
}
using Backend = Engine::Backend;
struct App {
@@ -152,6 +159,19 @@ int main(int argc, char** argv) {
auto& tcm = engine->getTransformManager();
auto ti = tcm.getInstance(app.mesh.renderable);
tcm.setTransform(ti, app.transform * mat4f::rotation(now, float3{ 0, 1, 0 }));
static int count = 0;
constexpr int allSize = 12000;
if (count++ == 5) {
for (size_t i = 0; i < allSize; ++i) {
auto mi = app.materialInstance = app.material->createInstance();
mi->setParameter("baseColor", RgbType::LINEAR, float3{0.8});
mi->setParameter("metallic", 1.0f);
mi->setParameter("roughness", 0.4f);
mi->setParameter("reflectance", 0.5f);
instances.push_back(mi);
}
}
});
FilamentApp::get().run(app.config, setup, cleanup);

View File

@@ -1,190 +0,0 @@
# Copyright (C) 2026 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/usr/bin/env python3
import argparse
import json
import os
import sys
import urllib.request
import urllib.error
import subprocess
# The base URL where historical size data is stored
BASE_URL = "https://raw.githubusercontent.com/google/filament-assets/main/sizeguard/"
def get_merge_base(target_branch="origin/main"):
"""Finds the merge base between HEAD and the target branch."""
try:
# Fetch the target branch to ensure we have the reference
result = subprocess.run(
["git", "merge-base", "HEAD", target_branch],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
text=True
)
return result.stdout.strip()
except subprocess.CalledProcessError:
print(f"Warning: Could not determine merge base with {target_branch}.", file=sys.stderr)
return None
def get_ancestors(start_commit, count=50):
"""Returns a list of ancestor commit hashes."""
try:
result = subprocess.run(
["git", "rev-list", f"--max-count={count}", start_commit],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
text=True
)
return result.stdout.strip().splitlines()
except subprocess.CalledProcessError as e:
print(f"Error listing ancestors: {e}", file=sys.stderr)
return []
def fetch_json(commit_hash):
"""Fetches the JSON file for the given commit from the assets repo."""
url = f"{BASE_URL}{commit_hash}.json"
try:
with urllib.request.urlopen(url) as response:
if response.status == 200:
print(f"Found historical data for commit {commit_hash}")
return json.loads(response.read().decode('utf-8'))
except urllib.error.HTTPError as e:
if e.code == 404:
return None
print(f"Warning: HTTP error fetching {url}: {e}", file=sys.stderr)
except Exception as e:
print(f"Warning: Error fetching {url}: {e}", file=sys.stderr)
return None
def flatten_data(data):
"""Flattens the JSON structure into a dictionary of name -> size."""
flat = {}
for item in data:
# Top level archive
flat[item['name']] = item['size']
# Inner content
if 'content' in item:
for inner in item['content']:
# Key format: ArchiveName/InnerName
key = f"{item['name']}/{inner['name']}"
flat[key] = inner['size']
return flat
def main():
parser = argparse.ArgumentParser(
description="Compare current artifact sizes against historical data."
)
parser.add_argument(
"current_json", help="Path to the JSON file generated for the current build."
)
parser.add_argument(
"--threshold", type=int, default=20480, help="Size increase threshold in bytes (default: 20KB)."
)
parser.add_argument(
"--target-branch", default="origin/main", help="The branch to compare against."
)
parser.add_argument(
"--artifacts", nargs="+",
help="List of artifact paths to check (e.g. 'foo.aar' or 'foo.aar/lib/arm64/bar.so')."
)
args = parser.parse_args()
if not os.path.exists(args.current_json):
print(f"Error: Current JSON file not found: {args.current_json}", file=sys.stderr)
sys.exit(1)
with open(args.current_json, 'r') as f:
current_data = json.load(f)
# 1. Find the starting commit (merge base)
start_commit = get_merge_base(args.target_branch)
if not start_commit:
print("Error: Could not determine a valid starting commit to search.", file=sys.stderr)
sys.exit(1)
print(f"Merge base with {args.target_branch} is {start_commit}")
# 2. Search for historical data
ancestors = get_ancestors(start_commit)
base_data = None
base_commit = None
for commit in ancestors:
base_data = fetch_json(commit)
if base_data:
base_commit = commit
break
if not base_data:
print(f"Warning: No historical size data found in the last {len(ancestors)} ancestors.",
file=sys.stderr)
sys.exit(0)
print(f"Comparing against historical data from commit {base_commit}")
# 3. Compare
current_flat = flatten_data(current_data)
base_flat = flatten_data(base_data)
failures = []
checked_count = 0
print(f"{'Artifact':<60} | {'Current':<10} | {'Base':<10} | {'Delta':<10} | {'Status'}")
print("-" * 110)
keys_to_check = args.artifacts if args.artifacts else current_flat.keys()
for name in keys_to_check:
if name not in current_flat:
print(f"Warning: Artifact '{name}' not found in current build output.", file=sys.stderr)
continue
current_size = current_flat[name]
checked_count += 1
status = "OK"
base_str = "N/A"
diff_str = "N/A"
if name in base_flat:
base_size = base_flat[name]
base_str = str(base_size)
diff = current_size - base_size
diff_str = f"{diff:+}"
if diff > args.threshold:
failures.append(name)
status = "FAIL"
else:
status = "NEW"
print(f"{name:<60} | {current_size:<10} | {base_str:<10} | {diff_str:<10} | {status}")
print("-" * 110)
if failures:
print(f"FAILURE: {len(failures)} artifacts exceeded threshold of {args.threshold} bytes.")
sys.exit(1)
else:
print(f"SUCCESS: {checked_count} artifacts checked. All within acceptable threshold.")
sys.exit(0)
if __name__ == "__main__":
main()