From c37d64e57457487e6f4f2332d5ac4f025b70b8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Mon, 19 Oct 2026 21:48:03 -0700 Subject: [PATCH] Updated stb_image_resize2. (#167) --- 3rdparty/stb/stb_image_resize2.h | 7942 ++++++++++++++++++++---------- 1 file changed, 5406 insertions(+), 2536 deletions(-) diff --git a/3rdparty/stb/stb_image_resize2.h b/3rdparty/stb/stb_image_resize2.h index 0798976..216637b 100644 --- a/3rdparty/stb/stb_image_resize2.h +++ b/3rdparty/stb/stb_image_resize2.h @@ -1,4 +1,4 @@ -/* stb_image_resize2 - v2.18 - public domain image resizing +/* stb_image_resize2 - v2.50 - public domain image resizing by Jeff Roberts (v2) and Jorge L Rodriguez http://github.com/nothings/stb @@ -61,14 +61,21 @@ after a resize. SIMD - Most of the routines have optimized SSE2, AVX, NEON and WASM versions. + Most of the routines have optimized SSE2, AVX, AVX-512, NEON and WASM versions. On Microsoft compilers, we automatically turn on SIMD for 64-bit x64 and ARM; for 32-bit x86 and ARM, you select SIMD mode by defining STBIR_SSE2 or - STBIR_NEON. For AVX and AVX2, we auto-select it by detecting the /arch:AVX + STBIR_NEON. + + For AVX and AVX2, we auto-select it by detecting the /arch:AVX or /arch:AVX2 switches. You can also always manually turn SSE2, AVX or AVX2 support on by defining STBIR_SSE2, STBIR_AVX or STBIR_AVX2. + For AVX-512, we turn it on by /arch:AVX512 (or -march=x86-64-v4, or the + -mavx512f -mavx512bw -mavx512vl trio on clang and GCC), or by defining + STBIR_AVX512. It's mostly useful on Zen 5 or modern Intel cores that don't + downclock on use. Best on very large resizes. + On Linux, SSE2 and Neon is on by default for 64-bit x64 or ARM64. For 32-bit, we select x86 SIMD mode by whether you have -msse2, -mavx or -mavx2 enabled on the command line. For 32-bit ARM, you must pass -mfpu=neon-vfpv4 for both @@ -87,8 +94,8 @@ timeframe), then fma will give you around a 15% speedup. You can force off SIMD in all cases by defining STBIR_NO_SIMD. You can turn - off AVX or AVX2 specifically with STBIR_NO_AVX or STBIR_NO_AVX2. AVX is 10% - to 40% faster, and AVX2 is generally another 12%. + off AVX, AVX2 or AVX-512 specifically with STBIR_NO_AVX, STBIR_NO_AVX2 or + STBIR_NO_AVX512. AVX is 10% to 40% faster, and AVX2 is generally another 12%. ALPHA CHANNEL Most of the resizing functions provide the ability to control how the alpha @@ -223,16 +230,50 @@ to clamp to on output, if that's important. MAX/MIN SCALE FACTORS - The input pixel resolutions are in integers, and we do the internal pointer - resolution in size_t sized integers. However, the scale ratio from input - resolution to output resolution is calculated in float form. This means - the effective possible scale ratio is limited to 24 bits (or 16 million - to 1). As you get close to the size of the float resolution (again, 16 - million pixels wide or high), you might start seeing float inaccuracy - issues in general in the pipeline. If you have to do extreme resizes, - you can usually do this is multiple stages (using float intermediate + Images can be up to 2^30 pixels wide or high (dimensions are ints, and + the internal pointer math is done in size_t). + + The positions of the filter taps are calculated in stbir_real (double by + default), which is accurate all the way to that limit. When the scale + works out to an exact fraction (which it does for most whole image + resizes), the positions are calculated with 64-bit integer math and are + exact at any size. + + If you are on a part without hardware doubles, you can define stbir_real + to float before you #include. Positions are then accurate to about 2^20 + pixels and are only off by half a pixel at 2^24, so for extreme resizes + on those parts, do it in multiple stages (using float intermediate buffers). + HUGE IMAGES AND TILING + Image strides are 64-bit, so the buffers themselves can be as large as + memory allows; widths and heights are limited to 2^30 (ints). To resize + an image larger than that in either direction, resize it in tiles: for + each output tile, point the resize at a piece of the input (the 64-bit + strides reach anywhere in the buffer) with the output tile's size, and + set the input region the tile maps to, in the piece's own coordinates, + with stbir_set_input_subrect. Two things keep the tiles seamless: + + - the piece must extend past the region by the filter's reach on every + side that isn't a real image edge (the filter's support in input + pixels: support x input / output when downsampling, support when + upsampling, plus a pixel of slack), because the edge modes act at + the piece's edges - a tap that runs off the piece is clamped or + wrapped there, as if the image ended. WRAP is the awkward one: a + tile at a real edge needs the pixels from the opposite edge of the + whole image, which a piece of the buffer can't reach, so you would + have to copy them into place next to the piece - it is simpler to + use another edge mode when tiling. + + - the region must be exactly the tile's share of the whole mapping + (tile size x whole input / whole output, at the tile's offset), so + every tile has the whole resize's scale and phase - a region drawn + from rounded numbers shifts that tile's samples against its + neighbors. + + A region that reaches past the source is clipped to it (the output + pixels that would map outside are not written). + FLIPPED IMAGES Stride is just the delta from one scanline to the next. This means you can use a negative stride to handle inverted images (point to the final @@ -251,71 +292,42 @@ NEW FILTERS A number of 1D filter kernels are supplied. For a list of supported filters, see the stbir_filter enum. You can install your own filters by - using the stbir_set_filter_callbacks function. + using the stbir_set_filter_callbacks_new function. PROGRESS For interactive use with slow resize operations, you can use the scanline callbacks in the extended API. It would have to be a *very* large image resample to need progress though - we're very fast. - CEIL and FLOOR - In scalar mode, the only functions we use from math.h are ceilf and floorf, - but if you have your own versions, you can define the STBIR_CEILF(v) and - STBIR_FLOORF(v) macros and we'll use them instead. In SIMD, we just use - our own versions. - ASSERT Define STBIR_ASSERT(boolval) to override assert() and not use assert.h - PORTING FROM VERSION 1 - The API has changed. You can continue to use the old version of stb_image_resize.h, - which is available in the "deprecated/" directory. + CACHE SIZE + A resize is run in column slices sized so that its working memory fits + the L2 cache. By default, the L2 size is read with cpuid on x86 (and a L2 + of a megabyte or more is treated as 512 KB, to keep the internal memory + down), and taken as 256 KB on other cpus. If that isn't right for you, + set stbir_l2_cache_size_override (bytes) before resizing, and it is used + as is; or define STBIR_ASK_WINDOWS_FOR_L2_SIZE, STBIR_ASK_LINUX_FOR_L2_SIZE + or STBIR_ASK_MACOS_FOR_L2_SIZE before the implementation include to ask + the OS instead of cpuid (see the comment above them for the headers each + one needs). Usually the defaults will just work fine. - If you're using the old simple-to-use API, porting is straightforward. - (For more advanced APIs, read the documentation.) - - stbir_resize_uint8(): - - call `stbir_resize_uint8_linear`, cast channel count to `stbir_pixel_layout` - - stbir_resize_float(): - - call `stbir_resize_float_linear`, cast channel count to `stbir_pixel_layout` - - stbir_resize_uint8_srgb(): - - function name is unchanged - - cast channel count to `stbir_pixel_layout` - - above is sufficient unless your image has alpha and it's not RGBA/BGRA - - in that case, follow the below instructions for stbir_resize_uint8_srgb_edgemode - - stbir_resize_uint8_srgb_edgemode() - - switch to the "medium complexity" API - - stbir_resize(), very similar API but a few more parameters: - - pixel_layout: cast channel count to `stbir_pixel_layout` - - data_type: STBIR_TYPE_UINT8_SRGB - - edge: unchanged (STBIR_EDGE_WRAP, etc.) - - filter: STBIR_FILTER_DEFAULT - - which channel is alpha is specified in stbir_pixel_layout, see enum for details + THREADING + To resize on several threads, call stbir_build_samplers_with_splits on + one thread with the number of threads you'd like, then call + stbir_resize_extended_split on each thread with its index (and a count + of 1), and stbir_free_samplers when they're all done. The build returns + the number of splits to actually use: it is limited to how many ways + the resize can be split, and then to the count the build estimates + will still speed things up (a small image or very narrow rows don't + get faster with more threads, they get slower). The estimate's constants + are statics in the implementation that you can override if you need to. FUTURE TODOS - * For polyphase integral filters, we just memcpy the coeffs to dupe - them, but we should indirect and use the same coeff memory. * Add pixel layout conversions for sensible different channel counts (maybe, 1->3/4, 3->4, 4->1, 3->1). - * For SIMD encode and decode scanline routines, do any pre-aligning - for bad input/output buffer alignments and pitch? - * For very wide scanlines, we should we do vertical strips to stay within - L2 cache. Maybe do chunks of 1K pixels at a time. There would be - some pixel reconversion, but probably dwarfed by things falling out - of cache. Probably also something possible with alternating between - scattering and gathering at high resize scales? - * Should we have a multiple MIPs at the same time function (could keep - more memory in cache during multiple resizes)? - * Rewrite the coefficient generator to do many at once. - * AVX-512 vertical kernels - worried about downclocking here. * Convert the reincludes to macros when we know they aren't changing. - * Experiment with pivoting the horizontal and always using the - vertical filters (which are faster, but perhaps not enough to overcome - the pivot cost and the extra memory touches). Need to buffer the whole - image so have to balance memory use. * Most of our code is internally function pointers, should we compile all the SIMD stuff always and dynamically dispatch? @@ -329,6 +341,52 @@ Nathan Reed: warning fixes for 1.0 REVISIONS + 2.50 (2026-10-07) Big performance release, especially for large and/or + threaded resizes! New stuff: + - Resizes now run in column slices sized to the L2 cache + (auto-detected, or set stbir_l2_cache_size_override), + which is 1.6-3x faster on large resizes (more when + threaded) and uses far less temp memory. + - Coefficient sets for rational scales are now shared, + using much less memory for these resizes (and faster). + - Rational downsamples now get faster dedicated kernels + (20-30% faster). + - Added horizontal gather kernels up to 17 taps, so + 4-tap filters stay on the fast gather path down to + 4:1 (20-30% faster than scatter path). Useful in mip + chain generation loops. + - For each resize, we adjust the offset of the internal + buffers to avoid 4K aliasing with the input buffer + (avoids random 40% slowdowns). + - New model for estimating vertical or horizontal first + mode. Much better at predicting best direction (lots + of resizes faster 30%-40% with this change). + - Added a model of thread addition improvement - if you + ask for 32 threads for example, the model will only use + the thread count that will improve performance (some + users were estimating this themselves - this should + be more accurate). + - 1 channel downsizes 30-40% faster (SIMD across samples). + - Added some new AVX kernels and optimizations. + - Added AVX-512 support - useful mostly just for Zen 5 + and recent Intel cores and large resamples. + - Bunch of cleanup and exactness in coefficient building, + along with new higher precision filter callbacks - this + moves our image resize limits up to 2^30. + - Added several new filters - more Mitchell variants, + some Lanczos, and the Muratori filter. + - Point sampling now actually does point sampling when + downsampling (previously only upsamples were point + sampled, down was averaged). + - Added 64-bit strides so that you can resize over the + 2^30 with manual tiling (calling resize multiple + times over the 64-bit area). + - Windows ARM64EC support (thanks navvyswethgraphics) + - Lots of little bugfixes in unusual corner cases. + NOTE THAT 2.50 IS NOT deterministic with prior versions + (among other improvements: coefficients calculated more + precisely, new v-first vs h-first decisions, etc). + 2.18 (2026-03-25) fixed coefficient calculation when skipping a coefficient off the left side of the window, added non-aligned access safe memcpy mode for scalar path, fixed various typos, and fixed @@ -394,12 +452,21 @@ typedef unsigned char stbir_uint8; typedef unsigned short stbir_uint16; typedef unsigned int stbir_uint32; typedef unsigned __int64 stbir_uint64; +typedef __int64 stbir_int64; #else #include typedef uint8_t stbir_uint8; typedef uint16_t stbir_uint16; typedef uint32_t stbir_uint32; typedef uint64_t stbir_uint64; +typedef int64_t stbir_int64; +#endif + +// we use this type for high precision floating point work - it's normally just a double +// but you can define it to float, if you don't have double support (some old embedded +// platforms). +#ifndef stbir_real +#define stbir_real double #endif #ifndef STBIRDEF @@ -420,7 +487,8 @@ typedef uint64_t stbir_uint64; // Easy-to-use API: // // * stride is the offset between successive rows of image data -// in memory, in bytes. specify 0 for packed continuously in memory +// in memory, in bytes (64-bit, so a buffer can be larger than 2 GB). +// specify 0 for packed continuously in memory // * colorspace is linear or sRGB as specified by function name // * Uses the default filters // * Uses edge mode clamped @@ -469,16 +537,16 @@ typedef enum // If output_pixels is NULL (0), then we will allocate the buffer and return it to you. //-------------------------------- -STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_type ); -STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_type ); -STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_type ); //=============================================================== @@ -502,14 +570,21 @@ typedef enum typedef enum { - STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses - STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios - STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering - STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque - STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline - STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3 - STBIR_FILTER_POINT_SAMPLE = 6, // Simple point sampling - STBIR_FILTER_OTHER = 7, // User callback specified + STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses + STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios + STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering + STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque + STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline (the default when upsampling) + STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3 (the default when downsampling) + STBIR_FILTER_POINT_SAMPLE = 6, // Simple point sampling + STBIR_FILTER_MITCHELL_QUARTER = 7, // Mitchell-Netrevalli with B=1/4, C=3/8 (Robidoux), a little sharper than Mitchell + STBIR_FILTER_MITCHELL_SIXTH = 8, // Mitchell-Netrevalli with B=1/6, C=5/12, sharper still + STBIR_FILTER_MITCHELL_NEG_SIXTH = 9, // Mitchell-Netrevalli with B=-1/6, C=7/12, over-sharpening + STBIR_FILTER_MITCHELL_NEG_THIRD = 10, // Mitchell-Netrevalli with B=-1/3, C=2/3, more over-sharpening + STBIR_FILTER_LANCZOS4 = 11, // Lanczos with 2 lobes (4 taps at 1:1), sharp, some ringing on hard edges + STBIR_FILTER_LANCZOS5 = 12, // Lanczos with 2.5 lobes (5 taps at 1:1), very sharp, often ringy + STBIR_FILTER_MURATORI6 = 13, // Casey Muratori's Bink 2 kernel (1 -4 19 19 -4 1)/32: sharper than Catmull-Rom, stable under repeated half pixel shifts when using 8-bit pixels + STBIR_FILTER_OTHER = 14, // User callback specified (keep last - the built-ins are everything below it) } stbir_filter; typedef enum @@ -523,14 +598,13 @@ typedef enum } stbir_datatype; // medium api -STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout, stbir_datatype data_type, stbir_edge edge, stbir_filter filter ); //=============================================================== - //=============================================================== // Extended-complexity API // @@ -561,12 +635,15 @@ STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int inpu // INPUT CALLBACK: this callback is used for input scanlines typedef void const * stbir_input_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ); -// OUTPUT CALLBACK: this callback is used for output scanlines -typedef void stbir_output_callback( void const * output_ptr, int num_pixels, int y, void * context ); +// OUTPUT CALLBACK: this callback is used for output scanlines (or pieces of scanlines - see below) +typedef void stbir_output_callback( void const * output_ptr, int num_pixels, int x, int y, void * context ); -// callbacks for user installed filters -typedef float stbir__kernel_callback( float x, float scale, void * user_data ); // centered at zero +// callbacks for user installed filters (there are old and new versions, please use the new versions from now on) +typedef float stbir__kernel_callback( float x, float scale, void * user_data ); typedef float stbir__support_callback( float scale, void * user_data ); +typedef stbir_real stbir__kernel_callback_new( stbir_real x, stbir_real scale, void * user_data ); +typedef stbir_real stbir__support_callback_new( stbir_real scale, void * user_data ); +typedef stbir_real stbir__radius_callback_new( stbir_real scale, stbir_real inv_scale, int is_upsample, void * user_data ); // internal structure with precomputed scaling typedef struct stbir__info stbir__info; @@ -576,14 +653,16 @@ typedef struct STBIR_RESIZE // use the stbir_resize_init and stbir_override fun void * user_data; void const * input_pixels; int input_w, input_h; - double input_s0, input_t0, input_s1, input_t1; + stbir_real input_s0, input_t0, input_s1, input_t1; + int input_s_n0, input_s_n1, input_s_d; // the input region as fractions, when it was set in pixels: s0 = n0/d, s1 = n1/d (d is 0 when set as reals) + int input_t_n0, input_t_n1, input_t_d; stbir_input_callback * input_cb; void * output_pixels; int output_w, output_h; int output_subx, output_suby, output_subw, output_subh; stbir_output_callback * output_cb; - int input_stride_in_bytes; - int output_stride_in_bytes; + stbir_int64 input_stride_in_bytes; + stbir_int64 output_stride_in_bytes; int splits; int fast_alpha; int needs_rebuild; @@ -594,8 +673,10 @@ typedef struct STBIR_RESIZE // use the stbir_resize_init and stbir_override fun stbir_datatype output_data_type; stbir_filter horizontal_filter, vertical_filter; stbir_edge horizontal_edge, vertical_edge; - stbir__kernel_callback * horizontal_filter_kernel; stbir__support_callback * horizontal_filter_support; + stbir__kernel_callback * horizontal_filter_kernel; stbir__support_callback * horizontal_filter_support; // old callback form stbir__kernel_callback * vertical_filter_kernel; stbir__support_callback * vertical_filter_support; + stbir__kernel_callback_new * horizontal_filter_kernel_new; stbir__support_callback_new * horizontal_filter_support_new; stbir__radius_callback_new * horizontal_filter_radius; + stbir__kernel_callback_new * vertical_filter_kernel_new; stbir__support_callback_new * vertical_filter_support_new; stbir__radius_callback_new * vertical_filter_radius; stbir__info * samplers; } STBIR_RESIZE; @@ -604,8 +685,8 @@ typedef struct STBIR_RESIZE // use the stbir_resize_init and stbir_override fun // First off, you must ALWAYS call stbir_resize_init on your resize structure before any of the other calls! STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize, - const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero + const void *input_pixels, int input_w, int input_h, stbir_int64 input_stride_in_bytes, // stride can be zero + void *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, // stride can be zero stbir_pixel_layout pixel_layout, stbir_datatype data_type ); //=============================================================== @@ -615,7 +696,7 @@ STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize, STBIRDEF void stbir_set_datatypes( STBIR_RESIZE * resize, stbir_datatype input_type, stbir_datatype output_type ); STBIRDEF void stbir_set_pixel_callbacks( STBIR_RESIZE * resize, stbir_input_callback * input_cb, stbir_output_callback * output_cb ); // no callbacks by default STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ); // pass back STBIR_RESIZE* by default -STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes ); +STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, stbir_int64 input_stride_in_bytes, void * output_pixels, stbir_int64 output_stride_in_bytes ); //=============================================================== @@ -628,10 +709,32 @@ STBIRDEF int stbir_set_pixel_layouts( STBIR_RESIZE * resize, stbir_pixel_layout STBIRDEF int stbir_set_edgemodes( STBIR_RESIZE * resize, stbir_edge horizontal_edge, stbir_edge vertical_edge ); // CLAMP by default STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_filter, stbir_filter vertical_filter ); // STBIR_DEFAULT_FILTER_UPSAMPLE/DOWNSAMPLE by default + +// Your own filters: per axis, a kernel callback and one of two ways to give its extent (a support or a radius - both +// on one axis returns 0; all null keeps the built-in filter). All in stbir_real (double unless user overridden to float). +// the resizer places the taps in stbir_real and rounds each weight to float once, into the coefficient table. +// +// The kernel returns the weight at x, in the filter's own pixels (output pixels when downsampling, input pixels when +// upsampling), centered on zero. It is also passed the ratio <= 1 (output / input downsampling, input / output +// upsampling), for kernels like the box whose shape depends on it. +// +// The support returns the filter's radius in those same filter pixels, and is passed the same ratio. For most kernels +// this is a constant (2 for the cubics, 1 for the triangle); the resizer converts it to input pixels itself. +// +// The radius is the alternative for a kernel whose extent depends on the scale: it returns the radius in INPUT pixels, +// from scale, inv_scale and the resizer's own up / down decision (is_upsample, 1:1 included), so an exact ratio gets +// an exact tap count (the built-in box is 0.5 * inv_scale + 0.5). +// +// static stbir_real my_kernel( stbir_real x, stbir_real scale, void * user ) { ... } +// static stbir_real my_support( stbir_real scale, void * user ) { return 2.0; } +// stbir_set_filter_callbacks_new( &resize, my_kernel, my_support, 0, my_kernel, my_support, 0 ); +STBIRDEF int stbir_set_filter_callbacks_new( STBIR_RESIZE * resize, stbir__kernel_callback_new * horizontal_filter, stbir__support_callback_new * horizontal_support, stbir__radius_callback_new * horizontal_radius, stbir__kernel_callback_new * vertical_filter, stbir__support_callback_new * vertical_support, stbir__radius_callback_new * vertical_radius ); + +// Old version of the filter callbacks (for backwards compatibility) - please use stbir_set_filter_callbacks_new for new code. STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support ); STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets both sub-regions (full regions by default) -STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ); // sets input sub-region (full region by default) +STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, stbir_real s0, stbir_real t0, stbir_real s1, stbir_real t1 ); // sets input sub-region (full region by default). a region past the image is clipped to it: the output pixels mapping outside aren't written STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets output sub-region (full region by default) // when inputting AND outputting non-premultiplied alpha pixels, we use a slower but higher quality technique @@ -664,14 +767,15 @@ STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize ); //=============================================================== // Use these functions for multithreading. // 1) You call stbir_build_samplers_with_splits first on the main thread -// 2) Then stbir_resize_with_split on each thread +// 2) Then stbir_resize_extended_split on each thread // 3) stbir_free_samplers when done on the main thread //-------------------------------- // This will build samplers for threading. // You can pass in the number of threads you'd like to use (try_splits). // It returns the number of splits (threads) that you can call it with. -/// It might be less if the image resize can't be split up that many ways. +// It might be less: when the image can't be split up that many ways, or when +// the build's estimate says more threads wouldn't help. STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int try_splits ); @@ -680,7 +784,7 @@ STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int try_sp // Note that you MUST call stbir_build_samplers_with_splits before stbir_resize_extended_split! -// Usually, you will always call stbir_resize_split with split_start as the thread_index +// Usually, you will always call stbir_resize_extended_split with split_start as the thread_index // and "1" for the split_count. // But, if you have a weird situation where you MIGHT want 8 threads, but sometimes // only 4 threads, you can use 0,2,4,6 for the split_start's and use "2" for the @@ -695,9 +799,10 @@ STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start //-------------------------------- // The input callback is super flexible - it calls you with the input address -// (based on the stride and base pointer), it gives you an optional_output -// pointer that you can fill, or you can just return your own pointer into -// your own data. +// of the start of the input row (based on the stride and base pointer), and the +// x position and pixel count that it needs from that row. It gives you an +// optional_output pointer that you can fill, or you can just return your own +// pointer into your own data. // // You can also do conversion from non-supported data types if necessary - in // this case, you ignore the input_ptr and just use the x and y parameters to @@ -711,7 +816,7 @@ STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start // void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ) // { // percentage_done = y / input_height; -// return input_ptr; // use buffer from call +// return ( (char*) input_ptr ) + x * pixel_width_in_bytes; // use buffer from call (input_ptr is the row start) // } // // Next example, copying: (copy from some other buffer or stream): @@ -730,20 +835,23 @@ STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start // // // The output callback is considerably simpler - it just calls you so that you can dump -// out each scanline. You could even directly copy out to disk if you have a simple format -// like TGA or BMP. You can also convert to other output types here if you want. +// out the output pixels. You can also convert to other output types here if you want. +// +// Note that the resizer can process the output in column slices (to stay in cache), so +// you are called with pieces of scanlines: num_pixels pixels starting at output pixel x +// on output row y. x and y are always in the full output image's pixel space (so with an +// output sub-rect, they include the sub-rect offset). The pieces of a scanline are not +// delivered in order, so if you want whole scanlines, assemble them yourself with x and y. // // Simple example: -// void const * my_output( void * output_ptr, int num_pixels, int y, void * context ) +// void my_output( void const * output_ptr, int num_pixels, int x, int y, void * context ) // { -// percentage_done = y / output_height; -// fwrite( output_ptr, pixel_width_in_bytes, num_pixels, output_file ); +// memcpy( ( (char*) my_output_image ) + y * my_output_stride + x * pixel_width_in_bytes, output_ptr, num_pixels * pixel_width_in_bytes ); // } //=============================================================== - //=============================================================== // optional built-in profiling API //-------------------------------- @@ -920,56 +1028,71 @@ typedef struct int widest; // widest single set of samples for an output } stbir__filter_extent_info; -typedef struct -{ - int n0; // First pixel of decode buffer to write to - int n1; // Last pixel of decode that will be written to - int pixel_offset_for_input; // Pixel offset into input_scanline -} stbir__span; - typedef struct stbir__scale_info { int input_full_size; int output_sub_size; - float scale; - float inv_scale; - float pixel_shift; // starting shift in output pixel space (in pixels) + stbir_real scale; + stbir_real inv_scale; + stbir_real pixel_shift; // starting shift in output pixel space (in pixels) int scale_is_rational; stbir_uint32 scale_numerator, scale_denominator; + stbir_real tap_scale_in, tap_scale_out; // 1 / 2N and 1 / 2D for stbir__tap_offset + int positions_exact; // the tap positions come from the exact fraction and an integer shift (stbir__tap_offset) + int shift_pixels; // pixel_shift as that integer + int approximated_drift; // the fraction approximates the scale: how far its far edge can land from the float chain's, in input pixels (0: exact) } stbir__scale_info; typedef struct { stbir__contributors * contributors; float* coefficients; - stbir__contributors * gather_prescatter_contributors; - float * gather_prescatter_coefficients; stbir__scale_info scale_info; - float support; stbir_filter filter_enum; - stbir__kernel_callback * filter_kernel; - stbir__support_callback * filter_support; + stbir__kernel_callback_new * filter_kernel_new; // the kernel: the built-in's or the user's, or + stbir__radius_callback_new * filter_radius; // the filter's extent, one of the three: a radius in input pixels (the box) or + stbir__support_callback_new * filter_support_new; // a support in filter pixels (the other built-ins); the user's, or the built-in's, + stbir__kernel_callback * filter_kernel_old; // an old back compat float kernel + stbir__support_callback * filter_support_old; // or an old back compat support function stbir_edge edge; int coefficient_width; + int entry_floats; // horizontal only: floats per packed column entry (STBIR__COLUMN_ENTRY_FLOATS of coefficient_width) + + // polyphase sharing (see stbir__plan_polyphase_sharing). The coefficient sets for the repeating part of a rational scale + // are stored once. 0 = every output has its own set. + int shared_period; // coefficient sets in the repeating block + int shared_prologue; // outputs with their own coefficient sets before it + int shared_epilogue; // outputs with their own coefficient sets after it + int shared_block_n0; // n0 of the block's first output (output P), and how far n0 moves per pass over the block + int shared_block_shift; + int shared_epilogue_n0; // n0 of the first epilogue output int filter_pixel_width; int filter_pixel_margin; int num_contributors; - int contributors_size; - int coefficients_size; + size_t contributors_size; + size_t coefficients_size; stbir__filter_extent_info extent_info; int is_gather; // 0 = scatter, 1 = gather with scale >= 1, 2 = gather with scale < 1 - int gather_prescatter_num_contributors; - int gather_prescatter_coefficient_width; - int gather_prescatter_contributors_size; - int gather_prescatter_coefficients_size; } stbir__sampler; +// horizontal routines combine the contribs and coeffs into one structure - coefficients is actually an array of width floats. typedef struct { - stbir__contributors conservative; - int edge_sizes[2]; // this can be less than filter_pixel_margin, if the filter and scaling falls off - stbir__span spans[2]; // can be two spans, if doing input subrect with clamp mode WRAP -} stbir__extents; + int advance; // how far the running decode pointer moves for the NEXT column, in floats (already times the channel count) + int width; // how many pixels this column reads (n1 - n0 + 1) - only the n-coefficient kernels look at it + float coefficients[ 1 ]; // width floats long, with a zero pad after for the widths whose kernels load one float past +} stbir__column_entry; + +// a column slice is a run of output columns that the whole vertical/horizontal loop is run over at once (instead of +// full scanlines), so the ring buffer and the temp buffers stay in cache. the coefficients are shared by all the +// slices, each slice just starts at a different offset into them. +typedef struct +{ + int x0; // first output pixel of the slice + int width; // output pixels in the slice + int in_n0, in_n1; // input pixels decoded for the slice (in the horizontal sampler's coordinate space - can hang off either edge with WRAP) + int first_n0; // input pixel output x0 starts reading at +} stbir__column_slice; typedef struct { @@ -988,6 +1111,7 @@ typedef struct int ring_buffer_begin_index; // first_scanline is at this index in the ring buffer int start_output_y, end_output_y; int start_input_y, end_input_y; // used in scatter only + int first_column_slice, num_column_slices; // the column slices this split runs: all of them when the splits are bands of rows, its own range when they are columns #ifdef STBIR__SEPARATE_ALLOCATIONS float** ring_buffers; // one pointer for each ring buffer @@ -997,22 +1121,28 @@ typedef struct float* vertical_buffer; + // where the buffers above were allocated. they each have a page of slack, and get repositioned within it at + // resize time to keep them clear of 4K aliasing with the input rows (see stbir__4K_alias_skew_buffers) + void* decode_buffer_alloc; + void* ring_buffer_alloc; + void* vertical_buffer_alloc; + char no_cache_straddle[64]; } stbir__per_split_info; -typedef float * stbir__decode_pixels_func( float * decode, int width_times_channels, void const * input ); -typedef void stbir__alpha_weight_func( float * decode_buffer, int width_times_channels ); +typedef float * stbir__decode_pixels_func( float * decode, ptrdiff_t width_times_channels, void const * input ); +typedef void stbir__alpha_weight_func( float * decode_buffer, ptrdiff_t width_times_channels ); typedef void stbir__horizontal_gather_channels_func( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, - stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ); -typedef void stbir__alpha_unweight_func(float * encode_buffer, int width_times_channels ); -typedef void stbir__encode_pixels_func( void * output, int width_times_channels, float const * encode ); + stbir__column_entry const * column, int entry_stride ); +typedef void stbir__alpha_unweight_func(float * encode_buffer, ptrdiff_t width_times_channels ); +typedef void stbir__encode_pixels_func( void * output, ptrdiff_t width_times_channels, float const * encode ); struct stbir__info { #ifdef STBIR_PROFILE union { - struct { stbir_uint64 total, build, alloc, horizontal, vertical, cleanup, pivot; } named; + struct { stbir_uint64 total, build, alloc, horizontal, vertical, cleanup, scatter; } named; stbir_uint64 array[7]; } profile; stbir_uint64 * current_zone_excluded_ptr; @@ -1023,9 +1153,9 @@ struct stbir__info void const * input_data; void * output_data; - int input_stride_bytes; - int output_stride_bytes; - int ring_buffer_length_bytes; // The length of an individual entry in the ring buffer. The total number of ring buffers is stbir__get_filter_pixel_width(filter) + stbir_int64 input_stride_bytes; + stbir_int64 output_stride_bytes; + size_t ring_buffer_length_bytes; // The length of an individual entry in the ring buffer (there are ring_buffer_num_entries of them) int ring_buffer_num_entries; // Total number of entries in the ring buffer. stbir_datatype input_type; @@ -1035,14 +1165,18 @@ struct stbir__info void * user_data; stbir_output_callback * out_pixels_cb; - stbir__extents scanline_extents; + stbir__contributors conservative_extents; // input pixel range that covers everything the horizontal filter can touch (decode buffers are sized from this) void * alloced_mem; stbir__per_split_info * split_info; // by default 1, but there will be N of these allocated based on the thread init you did + stbir__column_slice * column_slices; // the output columns, left to right (the last slice can be narrower) + int num_column_slices; + int split_cols, split_rows; // the split grid: splits = split_cols column groups x split_rows row bands (see stbir__split_mode) stbir__decode_pixels_func * decode_pixels; stbir__alpha_weight_func * alpha_weight; stbir__horizontal_gather_channels_func * horizontal_gather_channels; + stbir__horizontal_gather_channels_func * horizontal_gather_period1; // for the run over a polyphase period 1 set (integer downsamples: one coefficient set for every output), or null stbir__alpha_unweight_func * alpha_unweight; stbir__encode_pixels_func * encode_pixels; @@ -1053,6 +1187,7 @@ struct stbir__info stbir_internal_pixel_layout output_pixel_layout_internal; int input_color_and_type; + int input_callback_offset_bytes; // where the input callback's temp buffer sits relative to the decode target (see STBIR_INPUT_CALLBACK_PADDING) int offset_x, offset_y; // offset within output_data int vertical_first; int channels; @@ -1065,7 +1200,19 @@ struct stbir__info #define stbir__max_uint16_as_float 65535.0f #define stbir__max_uint8_as_float_inverted 3.9215689e-03f // (1.0f/255.0f) #define stbir__max_uint16_as_float_inverted 1.5259022e-05f // (1.0f/65535.0f) -#define stbir__small_float ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) + +// an alpha below this isn't divided by when un-premultiplying - alpha is effectively zero +#define stbir__alpha_too_small_to_divide ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) + +// a coefficient set whose sum is within this of 1 isn't renormalized: scaling by 1 / sum would move every float +// coefficient by less than half an ulp ( 2^-25 of itself ), so each would round straight back - the loop is a no-op +#define stbir__renorm_tolerance ( (stbir_real)1.0 / ( 1 << 26 ) ) + +// a kernel value this small (either sign) is dropped before the coefficients are normalized +#define stbir__too_small_coefficient ( (stbir_real)1.0 / ( 1 << 26 ) ) + +// a coefficient set whose sum is under this (either sign) is zeroed instead of normalized +#define stbir__too_small_coefficient_sum ( (stbir_real)1.0 / ( 1 << 16 ) ) // min/max friendly #define STBIR_CLAMP(x, xmin, xmax) for(;;) { \ @@ -1119,6 +1266,9 @@ typedef union // From https://gist.github.com/rygorous/2203834 +// the SIMD sRGB encoders index this with a float's bits munged into 912..1015, so this biases it back to the table size +#define STBIR__SRGB8_TAB4_BIAS ((127-13)*8) +#define STBIR__SRGB8_TAB4( index ) fp32_to_srgb8_tab4[ (ptrdiff_t)(index) - STBIR__SRGB8_TAB4_BIAS ] static const stbir_uint32 fp32_to_srgb8_tab4[104] = { 0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, 0x00a7001a, 0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, @@ -1162,17 +1312,16 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) } #ifndef STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT -#define STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT 32 // when downsampling and <= 32 scanlines of buffering, use gather. gather used down to 1/8th scaling for 25% win. +#define STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT 128 // when downsampling and <= 128 scanlines of buffering, use gather (scatter above that - 32 or more to 1 downsampling) #endif -#ifndef STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS -#define STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS 4 // when threading, what is the minimum number of scanlines for a split? -#endif -#define STBIR_INPUT_CALLBACK_PADDING 3 +// every scanline buffer carries this many extra floats past the end. we right justify into the buffer +// so the conversion can read ahead of the writes. also gives the horizontal kernels a one-float overread +#define STBIR_INPUT_CALLBACK_PADDING 16 #ifdef _M_IX86_FP -#if ( _M_IX86_FP >= 1 ) +#if ( _M_IX86_FP >= 2 ) #ifndef STBIR_SSE #define STBIR_SSE #endif @@ -1184,10 +1333,25 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define STBIR_NO_SIMD #endif -#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(_M_AMD64) || defined(__SSE2__) || defined(STBIR_SSE) || defined(STBIR_SSE2) +// (ARM64EC defines _M_X64 and _M_AMD64 as well as _M_ARM64EC, so the x64 tests have to leave it out - it is NEON below) +#if ( defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(_M_AMD64) || defined(__SSE2__) || defined(STBIR_SSE) || defined(STBIR_SSE2) ) && !defined(_M_ARM64EC) #ifndef STBIR_SSE2 #define STBIR_SSE2 #endif + // AVX-512: the foundation plus the byte/word (BW) and 128/256-bit (VL) extensions. /arch:AVX512 on microsoft, + // -mavx512f -mavx512bw -mavx512vl (or -march=x86-64-v4) on clang and gcc. Also implies AVX2 + #if ( defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VL__) ) || defined(STBIR_AVX512) + #ifdef STBIR_NO_AVX512 + #undef STBIR_AVX512 + #else + #ifndef STBIR_AVX512 + #define STBIR_AVX512 + #endif + #ifndef STBIR_AVX2 + #define STBIR_AVX2 + #endif + #endif + #endif #if defined(__AVX__) || defined(STBIR_AVX2) #ifndef STBIR_AVX #ifndef STBIR_NO_AVX @@ -1214,7 +1378,12 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #endif #endif -#if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || ((__ARM_NEON_FP & 4) != 0) || defined(__ARM_NEON__) +// 64-bit ARM (MSVC's ARM64EC included): the NEON code has a few AArch64 only forms +#if defined( _M_ARM64 ) || defined( _M_ARM64EC ) || defined( __aarch64__ ) || defined( __arm64__ ) +#define STBIR__ARM64 +#endif + +#if defined( STBIR__ARM64 ) || ((__ARM_NEON_FP & 4) != 0) || defined(__ARM_NEON__) #ifndef STBIR_NEON #define STBIR_NEON #endif @@ -1314,13 +1483,15 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_load( reg, ptr ) (reg) = _mm_loadu_ps( (float const*)(ptr) ) #define stbir__simdi_load( reg, ptr ) (reg) = _mm_loadu_si128 ( (stbir__simdi const*)(ptr) ) #define stbir__simdf_load1( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf) - #define stbir__simdi_load1( out, ptr ) (out) = _mm_castps_si128( _mm_load_ss( (float const*)(ptr) )) #define stbir__simdf_load1z( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values must be zero #define stbir__simdf_frep4( fvar ) _mm_set_ps1( fvar ) #define stbir__simdf_load1frep4( out, fvar ) (out) = _mm_set_ps1( fvar ) #define stbir__simdf_load2( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values can be random (not denormal or nan for perf) #define stbir__simdf_load2z( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values must be zero #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = _mm_castpd_ps(_mm_loadh_pd( _mm_castps_pd(reg), (double*)(ptr) )) + // pairwise add of two registers: out = ( a0+a1, a2+a3, b0+b1, b2+b3 ), and the halves version: ( a0+a2, a1+a3, b0+b2, b1+b3 ) + #define stbir__simdf_hadd2( out, a, b ) (out) = _mm_add_ps( _mm_shuffle_ps( a, b, (0<<0)+(2<<2)+(0<<4)+(2<<6) ), _mm_shuffle_ps( a, b, (1<<0)+(3<<2)+(1<<4)+(3<<6) ) ) + #define stbir__simdf_addhalves2( out, a, b ) (out) = _mm_add_ps( _mm_movelh_ps( a, b ), _mm_movehl_ps( b, a ) ) #define stbir__simdf_zeroP() _mm_setzero_ps() #define stbir__simdf_zero( reg ) (reg) = _mm_setzero_ps() @@ -1331,7 +1502,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_store2h( ptr, reg ) _mm_storeh_pd( (double*)(ptr), _mm_castps_pd(reg) ) #define stbir__simdi_store( ptr, reg ) _mm_storeu_si128( (__m128i*)(ptr), reg ) - #define stbir__simdi_store1( ptr, reg ) _mm_store_ss( (float*)(ptr), _mm_castsi128_ps(reg) ) #define stbir__simdi_store2( ptr, reg ) _mm_storel_epi64( (__m128i*)(ptr), (reg) ) #define stbir__prefetch( ptr ) _mm_prefetch((char*)(ptr), _MM_HINT_T0 ) @@ -1347,12 +1517,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) out3 = _mm_unpackhi_epi16( out3, zero ); \ } -#define stbir__simdi_expand_u8_to_1u32(out,ireg) \ - { \ - stbir__simdi zero = _mm_setzero_si128(); \ - out = _mm_unpacklo_epi8( ireg, zero ); \ - out = _mm_unpacklo_epi16( out, zero ); \ - } #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ { \ @@ -1362,7 +1526,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) } #define stbir__simdf_convert_float_to_i32( i, f ) (i) = _mm_cvttps_epi32(f) - #define stbir__simdf_convert_float_to_int( f ) _mm_cvtt_ss2si(f) #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),_mm_setzero_ps())))) #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())))) @@ -1373,31 +1536,24 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = _mm_mul_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) ) #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = _mm_mul_ss( reg, _mm_load_ss( (float const*)(ptr) ) ) #define stbir__simdf_add_mem( out, reg, ptr ) (out) = _mm_add_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) ) - #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = _mm_add_ss( reg, _mm_load_ss( (float const*)(ptr) ) ) #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd #include #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_fmadd_ps( mul1, mul2, add ) - #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_fmadd_ss( mul1, mul2, add ) #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ps( mul, _mm_loadu_ps( (float const*)(ptr) ), add ) #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ss( mul, _mm_load_ss( (float const*)(ptr) ), add ) #else #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_add_ps( add, _mm_mul_ps( mul1, mul2 ) ) - #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_add_ss( add, _mm_mul_ss( mul1, mul2 ) ) #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_add_ps( add, _mm_mul_ps( mul, _mm_loadu_ps( (float const*)(ptr) ) ) ) #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_add_ss( add, _mm_mul_ss( mul, _mm_load_ss( (float const*)(ptr) ) ) ) #endif #define stbir__simdf_add1( out, reg0, reg1 ) (out) = _mm_add_ss( reg0, reg1 ) - #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = _mm_mul_ss( reg0, reg1 ) #define stbir__simdf_and( out, reg0, reg1 ) (out) = _mm_and_ps( reg0, reg1 ) - #define stbir__simdf_or( out, reg0, reg1 ) (out) = _mm_or_ps( reg0, reg1 ) #define stbir__simdf_min( out, reg0, reg1 ) (out) = _mm_min_ps( reg0, reg1 ) #define stbir__simdf_max( out, reg0, reg1 ) (out) = _mm_max_ps( reg0, reg1 ) - #define stbir__simdf_min1( out, reg0, reg1 ) (out) = _mm_min_ss( reg0, reg1 ) - #define stbir__simdf_max1( out, reg0, reg1 ) (out) = _mm_max_ss( reg0, reg1 ) #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (3<<0) + (0<<2) + (1<<4) + (2<<6) ) ) #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (2<<0) + (3<<2) + (0<<4) + (1<<6) ) ) @@ -1471,9 +1627,17 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x } #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { STBIR__CONST_4_32i(x) } + #define STBIR__SIMDI_CONST4(var, a, b, c, d) stbir__simdi var = { STBIR__CONST_4d_32i(a, b, c, d) } #define STBIR__CONSTF(var) (var) #define STBIR__CONSTI(var) (var) + // an arbitrary byte permute (pshufb) needs SSSE3 + #if defined(STBIR_AVX) || defined(__SSSE3__) || defined(__SSE4_1__) + #include + #define STBIR_SIMD_BYTE_SHUFFLE + #define stbir__simdi_shuffle_bytes( out, reg, table ) (out) = _mm_shuffle_epi8( (reg), (table) ) + #endif + #if defined(STBIR_AVX) || defined(__SSE4_1__) #include #define stbir__simdf_pack_to_8words(out,reg0,reg1) out = _mm_packus_epi32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg0,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())), _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg1,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps()))) @@ -1529,12 +1693,11 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #ifdef STBIR_AVX2 + // vpmovzxbd takes eight bytes to eight dwords in one shuffle, so sixteen bytes are three (the high eight moved down first) #define stbir__simdi8_expand_u8_to_u32(out0,out1,ireg) \ { \ - stbir__simdi8 a, zero =_mm256_setzero_si256();\ - a = _mm256_permute4x64_epi64( _mm256_unpacklo_epi8( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), zero ),(0<<0)+(2<<2)+(1<<4)+(3<<6)); \ - out0 = _mm256_unpacklo_epi16( a, zero ); \ - out1 = _mm256_unpackhi_epi16( a, zero ); \ + out0 = _mm256_cvtepu8_epi32( ireg ); \ + out1 = _mm256_cvtepu8_epi32( _mm_unpackhi_epi64( ireg, ireg ) ); \ } #define stbir__simdf8_pack_to_16bytes(out,aa,bb) \ @@ -1542,8 +1705,8 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) stbir__simdi8 t; \ stbir__simdf8 af,bf; \ stbir__simdi8 a,b; \ - af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \ - bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \ + af = _mm256_min_ps( aa, STBIR_max_uint8_as_float8 ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint8_as_float8 ); \ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ a = _mm256_cvttps_epi32( af ); \ @@ -1552,14 +1715,17 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) out = _mm256_castsi256_si128( _mm256_permute4x64_epi64( _mm256_packus_epi16( t, t ), (0<<0)+(2<<2)+(1<<4)+(3<<6) ) ); \ } - #define stbir__simdi8_expand_u16_to_u32(out,ireg) out = _mm256_unpacklo_epi16( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), _mm256_setzero_si256() ); + #define stbir__simdi8_expand_u16_to_u32(out,ireg) out = _mm256_cvtepu16_epi32( ireg ); + + // a byte permute of 32 bytes by one 16 byte table, per 128-bit lane (vpshufb ymm) + #define stbir__simdi8_shuffle_bytes( out, reg, table ) (out) = _mm256_shuffle_epi8( (reg), _mm256_broadcastsi128_si256( table ) ) #define stbir__simdf8_pack_to_16words(out,aa,bb) \ { \ stbir__simdf8 af,bf; \ stbir__simdi8 a,b; \ - af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \ - bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \ + af = _mm256_min_ps( aa, STBIR_max_uint16_as_float8 ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint16_as_float8 ); \ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ a = _mm256_cvttps_epi32( af ); \ @@ -1583,8 +1749,8 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) stbir__simdi t; \ stbir__simdf8 af,bf; \ stbir__simdi8 a,b; \ - af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \ - bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \ + af = _mm256_min_ps( aa, STBIR_max_uint8_as_float8 ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint8_as_float8 ); \ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ a = _mm256_cvttps_epi32( af ); \ @@ -1609,8 +1775,8 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) stbir__simdi t0,t1; \ stbir__simdf8 af,bf; \ stbir__simdi8 a,b; \ - af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \ - bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \ + af = _mm256_min_ps( aa, STBIR_max_uint16_as_float8 ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint16_as_float8 ); \ af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ a = _mm256_cvttps_epi32( af ); \ @@ -1634,7 +1800,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) static __m256i stbir_00112233 = { STBIR__CONST_4d_32i( 0, 0, 1, 1 ), STBIR__CONST_4d_32i( 2, 2, 3, 3 ) }; #define stbir__simdf8_0123to00112233( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_00112233 ) - #define stbir__simdf8_add4( out, a8, b ) (out) = _mm256_add_ps( a8, _mm256_castps128_ps256( b ) ) + #define stbir__simdf8_add4( out, a8, b ) (out) = _mm256_add_ps( a8, _mm256_insertf128_ps( _mm256_setzero_ps(), b, 0 ) ) static __m256i stbir_load6 = { STBIR__CONST_4_32i( 0x80000000 ), STBIR__CONST_4d_32i( 0x80000000, 0x80000000, 0, 0 ) }; #define stbir__simdf8_load6z( out, ptr ) (out) = _mm256_maskload_ps( ptr, stbir_load6 ) @@ -1652,10 +1818,10 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf8_0123to11331133( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(1<<2)+(3<<4)+(3<<6) ) #define stbir__simdf8_0123to00220022( out, in ) (out) = _mm256_shuffle_ps ( in, in, (0<<0)+(0<<2)+(2<<4)+(2<<6) ) - #define stbir__simdf8_aaa1( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(1<<1)+(1<<2)+(0<<3)+(1<<4)+(1<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (3<<0) + (3<<2) + (3<<4) + (0<<6) ) - #define stbir__simdf8_1aaa( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(1<<2)+(1<<3)+(0<<4)+(1<<5)+(1<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (0<<4) + (0<<6) ) - #define stbir__simdf8_a1a1( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(0<<1)+(1<<2)+(0<<3)+(1<<4)+(0<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ) - #define stbir__simdf8_1a1a( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(0<<2)+(1<<3)+(0<<4)+(1<<5)+(0<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ) + #define stbir__simdf8_aaa1( out, alp, ones ) do { (out) = _mm256_blend_ps( alp, ones, (1<<0)+(1<<1)+(1<<2)+(0<<3)+(1<<4)+(1<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (3<<0) + (3<<2) + (3<<4) + (0<<6) ); } while(0) + #define stbir__simdf8_1aaa( out, alp, ones ) do { (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(1<<2)+(1<<3)+(0<<4)+(1<<5)+(1<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (0<<4) + (0<<6) ); } while(0) + #define stbir__simdf8_a1a1( out, alp, ones) do { (out) = _mm256_blend_ps( alp, ones, (1<<0)+(0<<1)+(1<<2)+(0<<3)+(1<<4)+(0<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ); } while(0) + #define stbir__simdf8_1a1a( out, alp, ones) do { (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(0<<2)+(1<<3)+(0<<4)+(1<<5)+(0<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ); } while(0) #define stbir__simdf8_zero( reg ) (reg) = _mm256_setzero_ps() @@ -1670,41 +1836,89 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #endif #define stbir__if_simdf8_cast_to_simdf4( val ) _mm256_castps256_ps128( val ) - #endif + static const stbir__simdf8 STBIR_max_uint16_as_float_inverted8 = { stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted }; + static const stbir__simdf8 STBIR_max_uint8_as_float_inverted8 = { stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted }; + static const stbir__simdf8 STBIR_ones8 = { 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 }; + static const stbir__simdf8 STBIR_simd_point58 = { 0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5 }; + static const stbir__simdf8 STBIR_max_uint8_as_float8 = { stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float, stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float }; + static const stbir__simdf8 STBIR_max_uint16_as_float8 = { stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float, stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float }; - #ifdef STBIR_FLOORF - #undef STBIR_FLOORF - #endif - #define STBIR_FLOORF stbir_simd_floorf - static stbir__inline float stbir_simd_floorf(float x) // martins floorf - { - #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41) - __m128 t = _mm_set_ss(x); - return _mm_cvtss_f32( _mm_floor_ss(t, t) ); - #else - __m128 f = _mm_set_ss(x); - __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f)); - __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(f, t), _mm_set_ss(-1.0f))); - return _mm_cvtss_f32(r); - #endif - } + // the 16-wide set (AVX-512) for the X-wide code. + #ifdef STBIR_AVX512 + #define STBIR_SIMD16 + #define stbir__simdf16 __m512 + #define stbir__simdi16 __m512i + #define stbir__simdf16_load( out, ptr ) (out) = _mm512_loadu_ps( (float const *)(ptr) ) + #define stbir__simdi16_load( out, ptr ) (out) = _mm512_loadu_si512( (void const *)(ptr) ) + #define stbir__simdf16_store( ptr, reg ) _mm512_storeu_ps( (float*)(ptr), reg ) + #define stbir__simdi16_store( ptr, reg ) _mm512_storeu_si512( (void*)(ptr), reg ) + #define stbir__simdf16_mult( out, a, b ) (out) = _mm512_mul_ps( (a), (b) ) + #define stbir__simdf16_add_mem( out, reg, ptr ) (out) = _mm512_add_ps( reg, _mm512_loadu_ps( (float const*)(ptr) ) ) + #define stbir__simdf16_min( out, a, b ) (out) = _mm512_min_ps( a, b ) + #define stbir__simdf16_max( out, a, b ) (out) = _mm512_max_ps( a, b ) + #define stbir__simdf16_zero( reg ) (reg) = _mm512_setzero_ps() + #define stbir__simdf16_frep16( fval ) _mm512_set1_ps( fval ) + #define stbir__simdi16_convert_i32_to_float( out, ireg ) (out) = _mm512_cvtepi32_ps( ireg ) + #define stbir__simdf16_convert_float_to_i32( i, f ) (i) = _mm512_cvttps_epi32( f ) + #define stbir__simdf16_cast_to_simdf4( val ) _mm512_castps512_ps128( val ) - #ifdef STBIR_CEILF - #undef STBIR_CEILF - #endif - #define STBIR_CEILF stbir_simd_ceilf - static stbir__inline float stbir_simd_ceilf(float x) // martins ceilf - { - #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41) - __m128 t = _mm_set_ss(x); - return _mm_cvtss_f32( _mm_ceil_ss(t, t) ); + #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd + #define stbir__simdf16_madd( out, add, mul1, mul2 ) (out) = _mm512_fmadd_ps( mul1, mul2, add ) + #define stbir__simdf16_madd_mem( out, add, mul, ptr ) (out) = _mm512_fmadd_ps( mul, _mm512_loadu_ps( (float const*)(ptr) ), add ) #else - __m128 f = _mm_set_ss(x); - __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f)); - __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(t, f), _mm_set_ss(1.0f))); - return _mm_cvtss_f32(r); + #define stbir__simdf16_madd( out, add, mul1, mul2 ) (out) = _mm512_add_ps( add, _mm512_mul_ps( mul1, mul2 ) ) + #define stbir__simdf16_madd_mem( out, add, mul, ptr ) (out) = _mm512_add_ps( add, _mm512_mul_ps( mul, _mm512_loadu_ps( (float const*)(ptr) ) ) ) #endif - } + + // the per lane shuffles: one 4-wide order, applied in every lane + #define stbir__simdf16_0123to0123( out, in ) (out) = (in) + #define stbir__simdf16_0123to1230( out, in ) (out) = _mm512_shuffle_ps( in, in, (1<<0)+(2<<2)+(3<<4)+(0<<6) ) + #define stbir__simdf16_0123to2103( out, in ) (out) = _mm512_shuffle_ps( in, in, (2<<0)+(1<<2)+(0<<4)+(3<<6) ) + #define stbir__simdf16_0123to3210( out, in ) (out) = _mm512_shuffle_ps( in, in, (3<<0)+(2<<2)+(1<<4)+(0<<6) ) + #define stbir__simdf16_0123to1032( out, in ) (out) = _mm512_shuffle_ps( in, in, (1<<0)+(0<<2)+(3<<4)+(2<<6) ) + #define stbir__simdf16_0123to3012( out, in ) (out) = _mm512_shuffle_ps( in, in, (3<<0)+(0<<2)+(1<<4)+(2<<6) ) + + // the alpha patterns: blend the ones in (the mask repeats per lane), then shuffle each lane + #define stbir__simdf16_aaa1( out, alp, ones ) do { (out) = _mm512_mask_blend_ps( (__mmask16)0x7777, alp, ones ); (out) = _mm512_shuffle_ps( out, out, (3<<0)+(3<<2)+(3<<4)+(0<<6) ); } while(0) + #define stbir__simdf16_1aaa( out, alp, ones ) do { (out) = _mm512_mask_blend_ps( (__mmask16)0xeeee, alp, ones ); (out) = _mm512_shuffle_ps( out, out, (1<<0)+(0<<2)+(0<<4)+(0<<6) ); } while(0) + #define stbir__simdf16_a1a1( out, alp, ones ) do { (out) = _mm512_mask_blend_ps( (__mmask16)0x5555, alp, ones ); (out) = _mm512_shuffle_ps( out, out, (1<<0)+(0<<2)+(3<<4)+(2<<6) ); } while(0) + #define stbir__simdf16_1a1a( out, alp, ones ) do { (out) = _mm512_mask_blend_ps( (__mmask16)0xaaaa, alp, ones ); (out) = _mm512_shuffle_ps( out, out, (1<<0)+(0<<2)+(3<<4)+(2<<6) ); } while(0) + + // integer conversions: sixteen bytes or words to sixteen dwords in one op, and the saturating narrows back + #define stbir__simdi16_expand_u8_to_u32( out, ireg ) (out) = _mm512_cvtepu8_epi32( ireg ) + #define stbir__simdi16_expand_u16_to_u32( out, ireg ) (out) = _mm512_cvtepu16_epi32( ireg ) + + #define stbir__simdf16_pack_to_32bytes( out, aa, bb ) \ + { \ + stbir__simdf16 af, bf; \ + af = _mm512_min_ps( aa, STBIR_max_uint8_as_float16 ); \ + bf = _mm512_min_ps( bb, STBIR_max_uint8_as_float16 ); \ + af = _mm512_max_ps( af, _mm512_setzero_ps() ); \ + bf = _mm512_max_ps( bf, _mm512_setzero_ps() ); \ + (out) = _mm256_inserti128_si256( _mm256_castsi128_si256( _mm512_cvtusepi32_epi8( _mm512_cvttps_epi32( af ) ) ), _mm512_cvtusepi32_epi8( _mm512_cvttps_epi32( bf ) ), 1 ); \ + } + + // the pack interleaves the two inputs per 128-bit lane, the 64-bit permute puts them back in order + static const stbir__simdi16 stbir__pack_lanes_in_order16 = { STBIR__CONST_4d_32i( 0,0, 2,0 ), STBIR__CONST_4d_32i( 4,0, 6,0 ), STBIR__CONST_4d_32i( 1,0, 3,0 ), STBIR__CONST_4d_32i( 5,0, 7,0 ) }; // 64-bit indexes 0 2 4 6 1 3 5 7 + #define stbir__simdf16_pack_to_32words( out, aa, bb ) \ + { \ + stbir__simdf16 af, bf; \ + af = _mm512_min_ps( aa, STBIR_max_uint16_as_float16 ); \ + bf = _mm512_min_ps( bb, STBIR_max_uint16_as_float16 ); \ + af = _mm512_max_ps( af, _mm512_setzero_ps() ); \ + bf = _mm512_max_ps( bf, _mm512_setzero_ps() ); \ + (out) = _mm512_permutexvar_epi64( stbir__pack_lanes_in_order16, _mm512_packus_epi32( _mm512_cvttps_epi32( af ), _mm512_cvttps_epi32( bf ) ) ); \ + } + + static const stbir__simdf16 STBIR_ones16 = { 1.0f,1.0f,1.0f,1.0f, 1.0f,1.0f,1.0f,1.0f, 1.0f,1.0f,1.0f,1.0f, 1.0f,1.0f,1.0f,1.0f }; + static const stbir__simdf16 STBIR_simd_point516 = { 0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f, 0.5f,0.5f,0.5f,0.5f }; + static const stbir__simdf16 STBIR_max_uint8_as_float16 = { 255.0f,255.0f,255.0f,255.0f, 255.0f,255.0f,255.0f,255.0f, 255.0f,255.0f,255.0f,255.0f, 255.0f,255.0f,255.0f,255.0f }; + static const stbir__simdf16 STBIR_max_uint16_as_float16 = { 65535.0f,65535.0f,65535.0f,65535.0f, 65535.0f,65535.0f,65535.0f,65535.0f, 65535.0f,65535.0f,65535.0f,65535.0f, 65535.0f,65535.0f,65535.0f,65535.0f }; + static const stbir__simdf16 STBIR_max_uint8_as_float_inverted16 = { stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted, stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted, stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted, stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted }; + static const stbir__simdf16 STBIR_max_uint16_as_float_inverted16 = { stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted, stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted, stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted, stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted }; + #endif // STBIR_AVX512 + + #endif #elif defined(STBIR_NEON) @@ -1718,14 +1932,16 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_load( reg, ptr ) (reg) = vld1q_f32( (float const*)(ptr) ) #define stbir__simdi_load( reg, ptr ) (reg) = vld1q_u32( (uint32_t const*)(ptr) ) - #define stbir__simdf_load1( out, ptr ) (out) = vld1q_dup_f32( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf) - #define stbir__simdi_load1( out, ptr ) (out) = vld1q_dup_u32( (uint32_t const*)(ptr) ) + #define stbir__simdf_load1( out, ptr ) (out) = vld1q_lane_f32( (float const*)(ptr), vdupq_n_f32(0), 0 ) // top values zero, like SSE (a replicated inf would turn into nan against the zero coefficient lanes) #define stbir__simdf_load1z( out, ptr ) (out) = vld1q_lane_f32( (float const*)(ptr), vdupq_n_f32(0), 0 ) // top values must be zero #define stbir__simdf_frep4( fvar ) vdupq_n_f32( fvar ) #define stbir__simdf_load1frep4( out, fvar ) (out) = vdupq_n_f32( fvar ) #define stbir__simdf_load2( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values can be random (not denormal or nan for perf) #define stbir__simdf_load2z( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values must be zero #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = vcombine_f32( vget_low_f32(reg), vld1_f32( (float const*)(ptr) ) ) + // pairwise add of two registers: out = ( a0+a1, a2+a3, b0+b1, b2+b3 ), and the halves version: ( a0+a2, a1+a3, b0+b2, b1+b3 ) + #define stbir__simdf_hadd2( out, a, b ) (out) = vcombine_f32( vpadd_f32( vget_low_f32( a ), vget_high_f32( a ) ), vpadd_f32( vget_low_f32( b ), vget_high_f32( b ) ) ) + #define stbir__simdf_addhalves2( out, a, b ) (out) = vaddq_f32( vcombine_f32( vget_low_f32( a ), vget_low_f32( b ) ), vcombine_f32( vget_high_f32( a ), vget_high_f32( b ) ) ) #define stbir__simdf_zeroP() vdupq_n_f32(0) #define stbir__simdf_zero( reg ) (reg) = vdupq_n_f32(0) @@ -1736,7 +1952,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_store2h( ptr, reg ) vst1_f32( (float*)(ptr), vget_high_f32(reg) ) #define stbir__simdi_store( ptr, reg ) vst1q_u32( (uint32_t*)(ptr), reg ) - #define stbir__simdi_store1( ptr, reg ) vst1q_lane_u32( (uint32_t*)(ptr), reg, 0 ) #define stbir__simdi_store2( ptr, reg ) vst1_u32( (uint32_t*)(ptr), vget_low_u32(reg) ) #define stbir__prefetch( ptr ) @@ -1751,11 +1966,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) out3 = vmovl_u16( vget_high_u16( h ) ); \ } - #define stbir__simdi_expand_u8_to_1u32(out,ireg) \ - { \ - uint16x8_t tmp = vmovl_u8( vget_low_u8( vreinterpretq_u8_u32(ireg) ) ); \ - out = vmovl_u16( vget_low_u16( tmp ) ); \ - } #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ { \ @@ -1765,7 +1975,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) } #define stbir__simdf_convert_float_to_i32( i, f ) (i) = vreinterpretq_u32_s32( vcvtq_s32_f32(f) ) - #define stbir__simdf_convert_float_to_int( f ) vgetq_lane_s32(vcvtq_s32_f32(f), 0) #define stbir__simdi_to_int( i ) (int)vgetq_lane_u32(i, 0) #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),vdupq_n_f32(0))), 0)) #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),vdupq_n_f32(0))), 0)) @@ -1775,30 +1984,29 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_f32( (float const*)(ptr) ) ) #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) ) #define stbir__simdf_add_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_f32( (float const*)(ptr) ) ) - #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) ) #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd (and also x64 no madd to arm madd) #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 ) - #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 ) #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_f32( (float const*)(ptr) ) ) #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_dup_f32( (float const*)(ptr) ) ) #else #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) ) - #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) ) #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_f32( (float const*)(ptr) ) ) ) #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_dup_f32( (float const*)(ptr) ) ) ) #endif #define stbir__simdf_add1( out, reg0, reg1 ) (out) = vaddq_f32( reg0, reg1 ) - #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = vmulq_f32( reg0, reg1 ) #define stbir__simdf_and( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vandq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) ) - #define stbir__simdf_or( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vorrq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) ) - #define stbir__simdf_min( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 ) - #define stbir__simdf_max( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 ) - #define stbir__simdf_min1( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 ) - #define stbir__simdf_max1( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 ) + // min / max with SSE's nan rule + #if defined( STBIR__ARM64 ) + #define stbir__simdf_min( out, reg0, reg1 ) (out) = vminnmq_f32( reg0, reg1 ) + #define stbir__simdf_max( out, reg0, reg1 ) (out) = vmaxnmq_f32( reg0, reg1 ) + #else + #define stbir__simdf_min( out, reg0, reg1 ) (out) = vbslq_f32( vcltq_f32( reg0, reg1 ), reg0, reg1 ) + #define stbir__simdf_max( out, reg0, reg1 ) (out) = vbslq_f32( vcgtq_f32( reg0, reg1 ), reg0, reg1 ) + #endif #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 3 ) #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 2 ) @@ -1806,7 +2014,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_a1a1( out, alp, ones ) (out) = vzipq_f32(vuzpq_f32(alp, alp).val[1], ones).val[0] #define stbir__simdf_1a1a( out, alp, ones ) (out) = vzipq_f32(ones, vuzpq_f32(alp, alp).val[0]).val[0] - #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) + #if defined( STBIR__ARM64 ) #define stbir__simdf_aaa1( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 3)), 3, ones, 3) #define stbir__simdf_1aaa( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 0)), 0, ones, 0) @@ -1831,6 +2039,10 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vqtbl1q_u8( vreinterpretq_u8_f32(reg), stbir_make16(one, two, three, four) ) ) #define stbir__simdf_swiz2( rega, regb, one, two, three, four ) vreinterpretq_f32_u8( vqtbl2q_u8( stbir_make16x2(rega,regb), stbir_make16(one, two, three, four) ) ) + // an arbitrary byte permute + #define STBIR_SIMD_BYTE_SHUFFLE + #define stbir__simdi_shuffle_bytes( out, reg, table ) (out) = vreinterpretq_u32_u8( vqtbl1q_u8( vreinterpretq_u8_u32( reg ), vreinterpretq_u8_u32( table ) ) ) + #define stbir__simdi_16madd( out, reg0, reg1 ) \ { \ int16x8_t r0 = vreinterpretq_s16_u32(reg0); \ @@ -1924,51 +2136,17 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #if defined( _MSC_VER ) && !defined(__clang__) #define STBIR__SIMDF_CONST(var, x) __declspec(align(8)) float var[] = { x, x, x, x } #define STBIR__SIMDI_CONST(var, x) __declspec(align(8)) uint32_t var[] = { x, x, x, x } + #define STBIR__SIMDI_CONST4(var, a, b, c, d) __declspec(align(8)) uint32_t var[] = { a, b, c, d } #define STBIR__CONSTF(var) (*(const float32x4_t*)var) #define STBIR__CONSTI(var) (*(const uint32x4_t*)var) #else #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x } #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x } + #define STBIR__SIMDI_CONST4(var, a, b, c, d) stbir__simdi var = { a, b, c, d } #define STBIR__CONSTF(var) (var) #define STBIR__CONSTI(var) (var) #endif - #ifdef STBIR_FLOORF - #undef STBIR_FLOORF - #endif - #define STBIR_FLOORF stbir_simd_floorf - static stbir__inline float stbir_simd_floorf(float x) - { - #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) - return vget_lane_f32( vrndm_f32( vdup_n_f32(x) ), 0); - #else - float32x2_t f = vdup_n_f32(x); - float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f)); - uint32x2_t a = vclt_f32(f, t); - uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(-1.0f)); - float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b))); - return vget_lane_f32(r, 0); - #endif - } - - #ifdef STBIR_CEILF - #undef STBIR_CEILF - #endif - #define STBIR_CEILF stbir_simd_ceilf - static stbir__inline float stbir_simd_ceilf(float x) - { - #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) - return vget_lane_f32( vrndp_f32( vdup_n_f32(x) ), 0); - #else - float32x2_t f = vdup_n_f32(x); - float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f)); - uint32x2_t a = vclt_f32(t, f); - uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(1.0f)); - float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b))); - return vget_lane_f32(r, 0); - #endif - } - #define STBIR_SIMD #elif defined(STBIR_WASM) @@ -1983,14 +2161,16 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) ) #define stbir__simdi_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) ) - #define stbir__simdf_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf) - #define stbir__simdi_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) ) + #define stbir__simdf_load1( out, ptr ) (out) = wasm_v128_load32_zero( (void const*)(ptr) ) // top values zero, like SSE (a replicated inf would turn into nan against the zero coefficient lanes) #define stbir__simdf_load1z( out, ptr ) (out) = wasm_v128_load32_zero( (void const*)(ptr) ) // top values must be zero #define stbir__simdf_frep4( fvar ) wasm_f32x4_splat( fvar ) #define stbir__simdf_load1frep4( out, fvar ) (out) = wasm_f32x4_splat( fvar ) - #define stbir__simdf_load2( out, ptr ) (out) = wasm_v128_load64_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdf_load2( out, ptr ) (out) = wasm_v128_load64_zero( (void const*)(ptr) ) // top values zero, like SSE #define stbir__simdf_load2z( out, ptr ) (out) = wasm_v128_load64_zero( (void const*)(ptr) ) // top values must be zero #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = wasm_v128_load64_lane( (void const*)(ptr), reg, 1 ) +// pairwise add of two registers: out = ( a0+a1, a2+a3, b0+b1, b2+b3 ), and the halves version: ( a0+a2, a1+a3, b0+b2, b1+b3 ) +#define stbir__simdf_hadd2( out, a, b ) (out) = wasm_f32x4_add( wasm_i32x4_shuffle( a, b, 0, 2, 4, 6 ), wasm_i32x4_shuffle( a, b, 1, 3, 5, 7 ) ) +#define stbir__simdf_addhalves2( out, a, b ) (out) = wasm_f32x4_add( wasm_i32x4_shuffle( a, b, 0, 1, 4, 5 ), wasm_i32x4_shuffle( a, b, 2, 3, 6, 7 ) ) #define stbir__simdf_zeroP() wasm_f32x4_const_splat(0) #define stbir__simdf_zero( reg ) (reg) = wasm_f32x4_const_splat(0) @@ -2001,7 +2181,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_store2h( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 1 ) #define stbir__simdi_store( ptr, reg ) wasm_v128_store( (void*)(ptr), reg ) - #define stbir__simdi_store1( ptr, reg ) wasm_v128_store32_lane( (void*)(ptr), reg, 0 ) #define stbir__simdi_store2( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 0 ) #define stbir__prefetch( ptr ) @@ -2016,11 +2195,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) out3 = wasm_u32x4_extend_high_u16x8( h ); \ } - #define stbir__simdi_expand_u8_to_1u32(out,ireg) \ - { \ - v128_t tmp = wasm_u16x8_extend_low_u8x16(ireg); \ - out = wasm_u32x4_extend_low_u16x8(tmp); \ - } #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ { \ @@ -2029,7 +2203,6 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) } #define stbir__simdf_convert_float_to_i32( i, f ) (i) = wasm_i32x4_trunc_sat_f32x4(f) - #define stbir__simdf_convert_float_to_int( f ) wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(f), 0) #define stbir__simdi_to_int( i ) wasm_i32x4_extract_lane(i, 0) #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint8_as_float),wasm_f32x4_const_splat(0))), 0)) #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint16_as_float),wasm_f32x4_const_splat(0))), 0)) @@ -2039,23 +2212,17 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load( (void const*)(ptr) ) ) #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load32_splat( (void const*)(ptr) ) ) #define stbir__simdf_add_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load( (void const*)(ptr) ) ) - #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load32_splat( (void const*)(ptr) ) ) #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) ) - #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) ) #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load( (void const*)(ptr) ) ) ) #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load32_splat( (void const*)(ptr) ) ) ) #define stbir__simdf_add1( out, reg0, reg1 ) (out) = wasm_f32x4_add( reg0, reg1 ) - #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = wasm_f32x4_mul( reg0, reg1 ) #define stbir__simdf_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 ) - #define stbir__simdf_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 ) #define stbir__simdf_min( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 ) #define stbir__simdf_max( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 ) - #define stbir__simdf_min1( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 ) - #define stbir__simdf_max1( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 ) #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 3, 4, 5, -1 ) #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 2, 3, 4, -1 ) @@ -2067,6 +2234,10 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_swiz( reg, one, two, three, four ) wasm_i32x4_shuffle(reg, reg, one, two, three, four) + // an arbitrary byte permute + #define STBIR_SIMD_BYTE_SHUFFLE + #define stbir__simdi_shuffle_bytes( out, reg, table ) (out) = wasm_i8x16_swizzle( reg, table ) + #define stbir__simdi_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 ) #define stbir__simdi_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 ) #define stbir__simdi_16madd( out, reg0, reg1 ) (out) = wasm_i32x4_dot_i16x8( reg0, reg1 ) @@ -2120,34 +2291,49 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) typedef float stbir__f32x4 __attribute__((__vector_size__(16), __aligned__(16))); #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = (v128_t)(stbir__f32x4){ x, x, x, x } #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x } + #define STBIR__SIMDI_CONST4(var, a, b, c, d) stbir__simdi var = { a, b, c, d } #define STBIR__CONSTF(var) (var) #define STBIR__CONSTI(var) (var) - #ifdef STBIR_FLOORF - #undef STBIR_FLOORF - #endif - #define STBIR_FLOORF stbir_simd_floorf - static stbir__inline float stbir_simd_floorf(float x) - { - return wasm_f32x4_extract_lane( wasm_f32x4_floor( wasm_f32x4_splat(x) ), 0); - } - - #ifdef STBIR_CEILF - #undef STBIR_CEILF - #endif - #define STBIR_CEILF stbir_simd_ceilf - static stbir__inline float stbir_simd_ceilf(float x) - { - return wasm_f32x4_extract_lane( wasm_f32x4_ceil( wasm_f32x4_splat(x) ), 0); - } - #define STBIR_SIMD #endif // SSE2/NEON/WASM #endif // NO SIMD -#ifdef STBIR_SIMD8 +// the X-wide names +#if defined(STBIR_SIMD16) + #define STBIR__U16_DECODE_BLOCK 16 // words per block in the u16 decoders (one 32 byte load) + #define stbir__simdfX stbir__simdf16 + #define stbir__simdiX stbir__simdi16 + #define stbir__simdfX_load stbir__simdf16_load + #define stbir__simdiX_load stbir__simdi16_load + #define stbir__simdfX_mult stbir__simdf16_mult + #define stbir__simdfX_add_mem stbir__simdf16_add_mem + #define stbir__simdfX_madd_mem stbir__simdf16_madd_mem + #define stbir__simdfX_store stbir__simdf16_store + #define stbir__simdiX_store stbir__simdi16_store + #define stbir__simdf_frepX stbir__simdf16_frep16 + #define stbir__simdfX_madd stbir__simdf16_madd + #define stbir__simdfX_min stbir__simdf16_min + #define stbir__simdfX_max stbir__simdf16_max + #define stbir__simdfX_aaa1 stbir__simdf16_aaa1 + #define stbir__simdfX_1aaa stbir__simdf16_1aaa + #define stbir__simdfX_a1a1 stbir__simdf16_a1a1 + #define stbir__simdfX_1a1a stbir__simdf16_1a1a + #define stbir__simdfX_convert_float_to_i32 stbir__simdf16_convert_float_to_i32 + #define stbir__simdfX_pack_to_words stbir__simdf16_pack_to_32words + #define stbir__simdfX_zero stbir__simdf16_zero + #define STBIR_onesX STBIR_ones16 + #define STBIR_max_uint8_as_floatX STBIR_max_uint8_as_float16 + #define STBIR_max_uint16_as_floatX STBIR_max_uint16_as_float16 + #define STBIR_simd_point5X STBIR_simd_point516 + #define stbir__simdfX_float_count 16 + #define stbir__simdfX_cast_to_simdf4( val ) stbir__simdf16_cast_to_simdf4( val ) + #define stbir__simdfX_0123to1230 stbir__simdf16_0123to1230 + #define stbir__simdfX_0123to2103 stbir__simdf16_0123to2103 +#elif defined(STBIR_SIMD8) + #define STBIR__U16_DECODE_BLOCK 8 #define stbir__simdfX stbir__simdf8 #define stbir__simdiX stbir__simdi8 #define stbir__simdfX_load stbir__simdf8_load @@ -2173,15 +2359,11 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define STBIR_max_uint16_as_floatX STBIR_max_uint16_as_float8 #define STBIR_simd_point5X STBIR_simd_point58 #define stbir__simdfX_float_count 8 + #define stbir__simdfX_cast_to_simdf4( val ) stbir__if_simdf8_cast_to_simdf4( val ) #define stbir__simdfX_0123to1230 stbir__simdf8_0123to12301230 #define stbir__simdfX_0123to2103 stbir__simdf8_0123to21032103 - static const stbir__simdf8 STBIR_max_uint16_as_float_inverted8 = { stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted }; - static const stbir__simdf8 STBIR_max_uint8_as_float_inverted8 = { stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted }; - static const stbir__simdf8 STBIR_ones8 = { 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 }; - static const stbir__simdf8 STBIR_simd_point58 = { 0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5 }; - static const stbir__simdf8 STBIR_max_uint8_as_float8 = { stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float, stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float }; - static const stbir__simdf8 STBIR_max_uint16_as_float8 = { stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float, stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float }; #else + #define STBIR__U16_DECODE_BLOCK 8 #define stbir__simdfX stbir__simdf #define stbir__simdiX stbir__simdi #define stbir__simdfX_load stbir__simdf_load @@ -2207,6 +2389,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define STBIR_max_uint8_as_floatX STBIR__CONSTF(STBIR_max_uint8_as_float) #define STBIR_max_uint16_as_floatX STBIR__CONSTF(STBIR_max_uint16_as_float) #define stbir__simdfX_float_count 4 + #define stbir__simdfX_cast_to_simdf4( val ) ( val ) #define stbir__if_simdf8_cast_to_simdf4( val ) ( val ) #define stbir__simdfX_0123to1230 stbir__simdf_0123to1230 #define stbir__simdfX_0123to2103 stbir__simdf_0123to2103 @@ -2428,7 +2611,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) absf_int = _mm_castps_si128(absf); // the cast is "free" (extra bypass latency, but no thruput hit) b_isnan = _mm_cmpunord_ps(absf, absf); // is this a NaN? b_isregular = _mm_cmpgt_epi32(f16max, absf_int); // (sub)normalized or special? - nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), c_nanbit); + nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), STBIR__CONSTI(c_nanbit)); inf_or_nan = _mm_or_si128(nanbit, STBIR__CONSTI(c_infty_as_fp16)); // output for specials b_issub = _mm_cmpgt_epi32(min_normal, absf_int); @@ -2457,7 +2640,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) stbir__simdi_store( output,final ); } -#elif defined(STBIR_NEON) && defined(_MSC_VER) && defined(_M_ARM64) && !defined(__clang__) // 64-bit ARM on MSVC (not clang) +#elif defined(STBIR_NEON) && defined(_MSC_VER) && defined(STBIR__ARM64) && !defined(__clang__) // 64-bit ARM on MSVC (not clang) static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) { @@ -2485,7 +2668,7 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0).n16_u16[0]; } -#elif defined(STBIR_NEON) && ( defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) ) // 64-bit ARM +#elif defined(STBIR_NEON) && defined( STBIR__ARM64 ) // 64-bit ARM static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) { @@ -2515,14 +2698,16 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) { - for (int i=0; i<8; i++) + int i; + for (i=0; i<8; i++) { output[i] = stbir__half_to_float(input[i]); } } static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input) { - for (int i=0; i<8; i++) + int i; + for (i=0; i<8; i++) { output[i] = stbir__float_to_half(input[i]); } @@ -2537,18 +2722,15 @@ static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) #define stbir__simdf_0123to2222( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,2,2 ) #define stbir__simdf_0123to1111( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,1,1 ) #define stbir__simdf_0123to0000( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,0 ) -#define stbir__simdf_0123to0003( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,3 ) #define stbir__simdf_0123to0001( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,1 ) #define stbir__simdf_0123to1122( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,2,2 ) #define stbir__simdf_0123to2333( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,3,3 ) -#define stbir__simdf_0123to0023( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,3 ) #define stbir__simdf_0123to1230( out, reg ) (out) = stbir__simdf_swiz( reg, 1,2,3,0 ) #define stbir__simdf_0123to2103( out, reg ) (out) = stbir__simdf_swiz( reg, 2,1,0,3 ) #define stbir__simdf_0123to3210( out, reg ) (out) = stbir__simdf_swiz( reg, 3,2,1,0 ) #define stbir__simdf_0123to2301( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,0,1 ) #define stbir__simdf_0123to3012( out, reg ) (out) = stbir__simdf_swiz( reg, 3,0,1,2 ) #define stbir__simdf_0123to0011( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,1,1 ) -#define stbir__simdf_0123to1100( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,0,0 ) #define stbir__simdf_0123to2233( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,3,3 ) #define stbir__simdf_0123to1133( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,3,3 ) #define stbir__simdf_0123to0022( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,2 ) @@ -2593,7 +2775,7 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) { char STBIR_SIMD_STREAMOUT_PTR (*) d = (char*) dest; char STBIR_SIMD_STREAMOUT_PTR( * ) d_end = ((char*) dest) + bytes; - ptrdiff_t ofs_to_src = (char*)src - (char*)dest; + char const * s = (char const*) src; // check overlaps STBIR_ASSERT( ( ( d >= ( (char*)src) + bytes ) ) || ( ( d + bytes ) <= (char*)src ) ); @@ -2608,8 +2790,9 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) do { STBIR_SIMD_NO_UNROLL(d); - d[ 0 ] = d[ ofs_to_src ]; + d[ 0 ] = s[ 0 ]; ++d; + ++s; } while ( d < d_end ); } } @@ -2617,8 +2800,9 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) { stbir__simdf x; // do one unaligned to get us aligned for the stream out below - stbir__simdf_load( x, ( d + ofs_to_src ) ); + stbir__simdf_load( x, s ); stbir__simdf_store( d, x ); + s += ( ( ( (size_t)d ) + 16 ) & ~15 ) - (size_t)d; d = (char*)( ( ( (size_t)d ) + 16 ) & ~15 ); STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR @@ -2630,12 +2814,14 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) { if ( d == d_end ) return; + s -= d - ( d_end - 16 ); d = d_end - 16; } - stbir__simdf_load( x, ( d + ofs_to_src ) ); + stbir__simdf_load( x, s ); stbir__simdf_store( d, x ); d += 16; + s += 16; } } } @@ -2644,14 +2830,15 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) stbir__simdfX x0,x1,x2,x3; // do one unaligned to get us aligned for the stream out below - stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count ); - stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count ); - stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count ); - stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count ); + stbir__simdfX_load( x0, s + 0*stbir__simdfX_float_count ); + stbir__simdfX_load( x1, s + 4*stbir__simdfX_float_count ); + stbir__simdfX_load( x2, s + 8*stbir__simdfX_float_count ); + stbir__simdfX_load( x3, s + 12*stbir__simdfX_float_count ); stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 ); stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 ); stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 ); stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 ); + s += ( ( ( (size_t)d ) + (16*stbir__simdfX_float_count) ) & ~((16*stbir__simdfX_float_count)-1) ) - (size_t)d; d = (char*)( ( ( (size_t)d ) + (16*stbir__simdfX_float_count) ) & ~((16*stbir__simdfX_float_count)-1) ); STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR @@ -2663,34 +2850,35 @@ static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) { if ( d == d_end ) return; + s -= d - ( d_end - (16*stbir__simdfX_float_count) ); d = d_end - (16*stbir__simdfX_float_count); } - stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count ); - stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count ); - stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count ); - stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count ); + stbir__simdfX_load( x0, s + 0*stbir__simdfX_float_count ); + stbir__simdfX_load( x1, s + 4*stbir__simdfX_float_count ); + stbir__simdfX_load( x2, s + 8*stbir__simdfX_float_count ); + stbir__simdfX_load( x3, s + 12*stbir__simdfX_float_count ); stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 ); stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 ); stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 ); stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 ); d += (16*stbir__simdfX_float_count); + s += (16*stbir__simdfX_float_count); } } } -// memcpy that is specically intentionally overlapping (src is smaller then dest, so can be -// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to -// the diff between dest and src) -static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes ) +// float copy that is specifically intentionally overlapping (src is below dest by less than the count, so a forward +// copy replicates the first floats through the rest. +static void stbir_overlapping_memcpy( float * dest, float const * src, size_t count ) { - char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src; - char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes; - ptrdiff_t ofs_to_dest = (char*)dest - (char*)src; + float STBIR_SIMD_STREAMOUT_PTR (*) sd = (float*) src; + float STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((float*) src) + count; + ptrdiff_t ofs_to_dest = dest - src; - if ( ofs_to_dest >= 16 ) // is the overlap more than 16 away? + if ( ofs_to_dest >= 4 ) // is the overlap more than 4 floats away? { - char STBIR_SIMD_STREAMOUT_PTR( * ) s_end16 = ((char*) src) + (bytes&~15); + float STBIR_SIMD_STREAMOUT_PTR( * ) s_end4 = ((float*) src) + (count&~(size_t)3); STBIR_SIMD_NO_UNROLL_LOOP_START do { @@ -2698,8 +2886,8 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte STBIR_SIMD_NO_UNROLL(sd); stbir__simdf_load( x, sd ); stbir__simdf_store( ( sd + ofs_to_dest ), x ); - sd += 16; - } while ( sd < s_end16 ); + sd += 4; + } while ( sd < s_end4 ); if ( sd == s_end ) return; @@ -2708,8 +2896,8 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte do { STBIR_SIMD_NO_UNROLL(sd); - *(int*)( sd + ofs_to_dest ) = *(int*) sd; - sd += 4; + sd[ ofs_to_dest ] = sd[ 0 ]; + ++sd; } while ( sd < s_end ); } @@ -2728,7 +2916,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #ifndef STBIR_PROFILE_FUNC -#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(__SSE2__) || defined(STBIR_SSE) || defined( _M_IX86_FP ) || defined(__i386) || defined( __i386__ ) || defined( _M_IX86 ) || defined( _X86_ ) +#if ( defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(__SSE2__) || defined(STBIR_SSE) || defined( _M_IX86_FP ) || defined(__i386) || defined( __i386__ ) || defined( _M_IX86 ) || defined( _X86_ ) ) && !defined(_M_ARM64EC) #ifdef _MSC_VER @@ -2746,10 +2934,14 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #endif // msvc -#elif defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || defined(__ARM_NEON__) +#elif defined( STBIR__ARM64 ) || defined(__ARM_NEON__) #if defined( _MSC_VER ) && !defined(__clang__) + #include // _ReadStatusReg, ARM64_SYSREG + #ifndef ARM64_CNTVCT // winnt.h defines it; without windows.h build the same encoding (CNTVCT_EL0, the generic timer counter) + #define ARM64_CNTVCT ARM64_SYSREG(3,3,14,0,2) + #endif #define STBIR_PROFILE_FUNC() _ReadStatusReg(ARM64_CNTVCT) #else @@ -2780,20 +2972,21 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte // super light-weight micro profiler #define STBIR_PROFILE_START_ll( info, wh ) { stbir_uint64 wh##thiszonetime = STBIR_PROFILE_FUNC(); stbir_uint64 * wh##save_parent_excluded_ptr = info->current_zone_excluded_ptr; stbir_uint64 wh##current_zone_excluded = 0; info->current_zone_excluded_ptr = &wh##current_zone_excluded; #define STBIR_PROFILE_END_ll( info, wh ) wh##thiszonetime = STBIR_PROFILE_FUNC() - wh##thiszonetime; info->profile.named.wh += wh##thiszonetime - wh##current_zone_excluded; *wh##save_parent_excluded_ptr += wh##thiszonetime; info->current_zone_excluded_ptr = wh##save_parent_excluded_ptr; } -#define STBIR_PROFILE_FIRST_START_ll( info, wh ) { int i; info->current_zone_excluded_ptr = &info->profile.named.total; for(i=0;iprofile.array);i++) info->profile.array[i]=0; } STBIR_PROFILE_START_ll( info, wh ); -#define STBIR_PROFILE_CLEAR_EXTRAS_ll( info, num ) { int extra; for(extra=1;extra<(num);extra++) { int i; for(i=0;iprofile.array);i++) (info)[extra].profile.array[i]=0; } } +#define STBIR_PROFILE_FIRST_START_ll( info, wh ) { int zone; info->current_zone_excluded_ptr = &info->profile.named.total; for(zone=0;zone<(int)STBIR__ARRAY_SIZE(info->profile.array);zone++) info->profile.array[zone]=0; } STBIR_PROFILE_START_ll( info, wh ); +#define STBIR_PROFILE_CLEAR_EXTRAS_ll( info, num ) { int extra; for(extra=1;extra<(num);extra++) { int zone; for(zone=0;zone<(int)STBIR__ARRAY_SIZE((info)->profile.array);zone++) (info)[extra].profile.array[zone]=0; } } // for thread data #define STBIR_PROFILE_START( wh ) STBIR_PROFILE_START_ll( split_info, wh ) #define STBIR_PROFILE_END( wh ) STBIR_PROFILE_END_ll( split_info, wh ) #define STBIR_PROFILE_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( split_info, wh ) #define STBIR_PROFILE_CLEAR_EXTRAS() STBIR_PROFILE_CLEAR_EXTRAS_ll( split_info, split_count ) +#define STBIR_PROFILE_NEST_IN_FIRST( extra ) { if ( ( extra ) != split_info ) ( extra )->current_zone_excluded_ptr = split_info->current_zone_excluded_ptr; } // an extra split's zones count against the first split's open zone // for build data #define STBIR_PROFILE_BUILD_START( wh ) STBIR_PROFILE_START_ll( profile_info, wh ) #define STBIR_PROFILE_BUILD_END( wh ) STBIR_PROFILE_END_ll( profile_info, wh ) #define STBIR_PROFILE_BUILD_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( profile_info, wh ) -#define STBIR_PROFILE_BUILD_CLEAR( info ) { int i; for(i=0;iprofile.array);i++) info->profile.array[i]=0; } +#define STBIR_PROFILE_BUILD_CLEAR( info ) { int zone; for(zone=0;zone<(int)STBIR__ARRAY_SIZE(info->profile.array);zone++) info->profile.array[zone]=0; } #else // no profile @@ -2807,6 +3000,7 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #define STBIR_PROFILE_END( wh ) #define STBIR_PROFILE_FIRST_START( wh ) #define STBIR_PROFILE_CLEAR_EXTRAS( ) +#define STBIR_PROFILE_NEST_IN_FIRST( extra ) #define STBIR_PROFILE_BUILD_START( wh ) #define STBIR_PROFILE_BUILD_END( wh ) @@ -2815,17 +3009,6 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #endif // stbir_profile -#ifndef STBIR_CEILF -#include -#if _MSC_VER <= 1200 // support VC6 for Sean -#define STBIR_CEILF(x) ((float)ceil((float)(x))) -#define STBIR_FLOORF(x) ((float)floor((float)(x))) -#else -#define STBIR_CEILF(x) ceilf(x) -#define STBIR_FLOORF(x) floorf(x) -#endif -#endif - #ifndef STBIR_MEMCPY // For memcpy #include @@ -2834,43 +3017,29 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte #ifndef STBIR_SIMD -// memcpy that is specifically intentionally overlapping (src is smaller then dest, so can be -// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to -// the diff between dest and src) -static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes ) +// float copy that is specifically intentionally overlapping (src is below dest by less than the count, so a forward +// copy replicates the first floats through the rest. +static void stbir_overlapping_memcpy( float * dest, float const * src, size_t count ) { - char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src; - char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes; - ptrdiff_t ofs_to_dest = (char*)dest - (char*)src; + float STBIR_SIMD_STREAMOUT_PTR (*) sd = (float*) src; + float STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((float*) src) + count; + ptrdiff_t ofs_to_dest = dest - src; - if ( ofs_to_dest >= 8 ) // is the overlap more than 8 away + if ( ofs_to_dest >= 2 ) // is the overlap more than 2 floats away { - char STBIR_SIMD_STREAMOUT_PTR( * ) s_end8 = ((char*) src) + (bytes&~7); + float STBIR_SIMD_STREAMOUT_PTR( * ) s_end2 = ((float*) src) + (count&~(size_t)1); - if ( ( ( ((ptrdiff_t)dest)|((ptrdiff_t)src) ) & 7 ) == 0 ) // is it 8byte aligned? + STBIR_NO_UNROLL_LOOP_START + do { - STBIR_NO_UNROLL_LOOP_START - do - { - STBIR_NO_UNROLL(sd); - *(stbir_uint64*)( sd + ofs_to_dest ) = *(stbir_uint64*) sd; - sd += 8; - } while ( sd < s_end8 ); - } - else - { - STBIR_NO_UNROLL_LOOP_START - do - { - int a,b; - STBIR_NO_UNROLL(sd); - a = ((int*)sd)[0]; - b = ((int*)sd)[1]; - ((int*)( sd + ofs_to_dest ))[0] = a; - ((int*)( sd + ofs_to_dest ))[1] = b; - sd += 8; - } while ( sd < s_end8 ); - } + float a,b; + STBIR_NO_UNROLL(sd); + a = sd[0]; + b = sd[1]; + sd[ ofs_to_dest ] = a; + sd[ ofs_to_dest + 1 ] = b; + sd += 2; + } while ( sd < s_end2 ); if ( sd == s_end ) return; @@ -2880,155 +3049,406 @@ static void stbir_overlapping_memcpy( void * dest, void const * src, size_t byte do { STBIR_NO_UNROLL(sd); - *(int*)( sd + ofs_to_dest ) = *(int*) sd; - sd += 4; + sd[ ofs_to_dest ] = sd[ 0 ]; + ++sd; } while ( sd < s_end ); } #endif -static float stbir__filter_trapezoid(float x, float scale, void * user_data) +// the built-in kernels (all in stbir_real; the generator rounds the weight to float once, on the way into the +// coefficient table). The box filter as the overlap of a pixel with the output pixel's footprint: 1 within +// 0.5 - scale / 2 of the center, 0 beyond 0.5 + scale / 2, a ramp between. +static stbir_real stbir__filter_trapezoid(stbir_real x, stbir_real scale, void * user_data) { - float halfscale = scale / 2; - float t = 0.5f + halfscale; - STBIR_ASSERT(scale <= 1); + stbir_real halfscale = scale * (stbir_real)0.5; + stbir_real t = (stbir_real)0.5 + halfscale; + stbir_real r = (stbir_real)0.5 - halfscale; + STBIR_ASSERT(scale <= (stbir_real)1.0); STBIR__UNUSED(user_data); - if ( x < 0.0f ) x = -x; + if ( x < (stbir_real) 0.0 ) x = -x; - if (x >= t) - return 0.0f; - else - { - float r = 0.5f - halfscale; - if (x <= r) - return 1.0f; - else - return (t - x) / scale; - } + if ( x >= t ) + return (stbir_real)0.0; + if ( x <= r ) + return (stbir_real)1.0; + return ( t - x ) / scale; } -static float stbir__support_trapezoid(float scale, void * user_data) +// the box's extent depends on the scale, so it is given as a radius (in input pixels): one multiply, so an integer +// ratio's tap count is exact +static stbir_real stbir__radius_trapezoid(stbir_real scale, stbir_real inv_scale, int is_upsample, void * user_data) { + STBIR__UNUSED(scale); + STBIR__UNUSED(is_upsample); STBIR__UNUSED(user_data); - return 0.5f + scale / 2.0f; + return (stbir_real)0.5 * inv_scale + (stbir_real)0.5; // ( 0.5 + scale / 2 ) output pixels downsampling, 0.5 + inv_scale / 2 input pixels upsampling: the same number } -static float stbir__filter_triangle(float x, float s, void * user_data) +static stbir_real stbir__filter_triangle(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - if ( x < 0.0f ) x = -x; + if ( x < (stbir_real) 0.0 ) x = -x; - if (x <= 1.0f) - return 1.0f - x; + if (x <= (stbir_real) 1.0) + return (stbir_real) 1.0 - x; else - return 0.0f; + return (stbir_real) 0.0; } -static float stbir__filter_point(float x, float s, void * user_data) +static stbir_real stbir__filter_point(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(x); STBIR__UNUSED(s); STBIR__UNUSED(user_data); - return 1.0f; + return (stbir_real) 1.0; } -static float stbir__filter_cubic(float x, float s, void * user_data) +// the Mitchell-Netrevalli cubics with C = (1-B)/2: B=1 is the cubic b-spline, 1/3 Mitchell, 0 Catmull-Rom +static stbir__inline stbir_real stbir__mitchell_netrevalli(stbir_real x, stbir_real B) +{ + if ( x < (stbir_real) 0.0 ) x = -x; + + if (x < (stbir_real) 1.0) + return (((stbir_real) 6.0 - (stbir_real) 2.0*B) + x*x*(((stbir_real) 9.0 - (stbir_real) 6.0*B)*x - ((stbir_real) 15.0 - (stbir_real) 9.0*B)))/(stbir_real) 6.0; + else if (x < (stbir_real) 2.0) + return (((stbir_real) 12.0 - (stbir_real) 4.0*B) + x*(((stbir_real) 12.0*B - (stbir_real) 24.0) + x*(((stbir_real) 15.0 - (stbir_real) 9.0*B) + ((stbir_real) 2.0*B - (stbir_real) 3.0)*x)))/(stbir_real) 6.0; + + return (stbir_real) 0.0; +} + +static stbir_real stbir__filter_cubic(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - if ( x < 0.0f ) x = -x; - - if (x < 1.0f) - return (4.0f + x*x*(3.0f*x - 6.0f))/6.0f; - else if (x < 2.0f) - return (8.0f + x*(-12.0f + x*(6.0f - x)))/6.0f; - - return (0.0f); + return stbir__mitchell_netrevalli( x, (stbir_real) 1.0 ); } -static float stbir__filter_catmullrom(float x, float s, void * user_data) +// Catmull-Rom is the family at B=0, written out (it's the upsampling default) +static stbir_real stbir__filter_catmullrom(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - if ( x < 0.0f ) x = -x; + if ( x < (stbir_real) 0.0 ) x = -x; - if (x < 1.0f) - return 1.0f - x*x*(2.5f - 1.5f*x); - else if (x < 2.0f) - return 2.0f - x*(4.0f + x*(0.5f*x - 2.5f)); + if (x < (stbir_real) 1.0) + return (stbir_real) 1.0 - x*x*((stbir_real) 2.5 - (stbir_real) 1.5*x); + else if (x < (stbir_real) 2.0) + return (stbir_real) 2.0 - x*((stbir_real) 4.0 + x*((stbir_real) 0.5*x - (stbir_real) 2.5)); - return (0.0f); + return (stbir_real) 0.0; } -static float stbir__filter_mitchell(float x, float s, void * user_data) +static stbir_real stbir__filter_mitchell(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - if ( x < 0.0f ) x = -x; - - if (x < 1.0f) - return (16.0f + x*x*(21.0f * x - 36.0f))/18.0f; - else if (x < 2.0f) - return (32.0f + x*(-60.0f + x*(36.0f - 7.0f*x)))/18.0f; - - return (0.0f); + return stbir__mitchell_netrevalli( x, (stbir_real) 1.0 / (stbir_real) 3.0 ); } -static float stbir__support_zeropoint5(float s, void * user_data) +static stbir_real stbir__filter_mitchell_quarter(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - return 0.5f; + + return stbir__mitchell_netrevalli( x, (stbir_real) 0.25 ); } -static float stbir__support_one(float s, void * user_data) +static stbir_real stbir__filter_mitchell_sixth(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - return 1; + + return stbir__mitchell_netrevalli( x, (stbir_real) 1.0 / (stbir_real) 6.0 ); } -static float stbir__support_two(float s, void * user_data) +static stbir_real stbir__filter_mitchell_neg_sixth(stbir_real x, stbir_real s, void * user_data) { STBIR__UNUSED(s); STBIR__UNUSED(user_data); - return 2; + + return stbir__mitchell_netrevalli( x, (stbir_real) -1.0 / (stbir_real) 6.0 ); +} + +static stbir_real stbir__filter_mitchell_neg_third(stbir_real x, stbir_real s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + return stbir__mitchell_netrevalli( x, (stbir_real) -1.0 / (stbir_real) 3.0 ); +} + +// sin(pi x) / (pi x), done here so the coefficients match on every platform (no libm). sin(pi x) flips sign at every +// whole x, so reduce to the nearest whole and run the series on [-pi/2, pi/2] (good to ~1e-9). +static stbir_real stbir__sinc_pi(stbir_real x) +{ + stbir_real const pi = (stbir_real) 3.14159265358979323846; + stbir_real u, u2, s; + int n; + + if ( x < (stbir_real) 0.0 ) x = -x; + + if (x < (stbir_real) 1e-4) + { + stbir_real px = pi * x; + return (stbir_real) 1.0 - px*px / (stbir_real) 6.0; + } + + n = (int) ( x + (stbir_real) 0.5 ); + u = pi * ( x - (stbir_real) n ); + u2 = u*u; + s = (stbir_real) 1.0 - u2 / (stbir_real) 156.0; // the series from the inside out: 1 - u^2 / ( 2k ( 2k + 1 ) ) + s = (stbir_real) 1.0 - u2 / (stbir_real) 110.0 * s; + s = (stbir_real) 1.0 - u2 / (stbir_real) 72.0 * s; + s = (stbir_real) 1.0 - u2 / (stbir_real) 42.0 * s; + s = (stbir_real) 1.0 - u2 / (stbir_real) 20.0 * s; + s = (stbir_real) 1.0 - u2 / (stbir_real) 6.0 * s; + s = u * s; + if ( n & 1 ) s = -s; + + return s / ( pi * x ); +} + +// Lanczos: a sinc windowed by a sinc stretched to the support +static stbir_real stbir__filter_lanczos(stbir_real x, stbir_real support) +{ + if ( x < (stbir_real) 0.0 ) x = -x; + + if (x < support) + return stbir__sinc_pi( x ) * stbir__sinc_pi( x / support ); + + return (stbir_real) 0.0; +} + +static stbir_real stbir__filter_lanczos4(stbir_real x, stbir_real s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + return stbir__filter_lanczos( x, (stbir_real) 2.0 ); +} + +static stbir_real stbir__filter_lanczos5(stbir_real x, stbir_real s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + return stbir__filter_lanczos( x, (stbir_real) 2.5 ); +} + +// Casey Muratori's Bink 2 half pixel kernel ( 1 -4 19 19 -4 1 ) / 32: a Catmull-Rom spline through knots every half +// pixel (1 at 0, the weights at the half pixels, 0 at the whole ones), padded at both ends so the four points stay in the table +static stbir_real stbir__filter_muratori6(stbir_real x, stbir_real s, void * user_data) +{ + static stbir_real const knots[9] = { (stbir_real) 19.0/(stbir_real) 32.0, (stbir_real) 1.0, (stbir_real) 19.0/(stbir_real) 32.0, (stbir_real) 0.0, (stbir_real) -4.0/(stbir_real) 32.0, (stbir_real) 0.0, (stbir_real) 1.0/(stbir_real) 32.0, (stbir_real) 0.0, (stbir_real) 0.0 }; + stbir_real t, p0, p1, p2, p3; + int i; + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + if ( x < (stbir_real) 0.0 ) x = -x; + + if ( x >= (stbir_real) 3.0 ) + return (stbir_real) 0.0; + + t = x * (stbir_real) 2.0; + i = (int) t; + t -= (stbir_real) i; + + p0 = knots[i]; + p1 = knots[i+1]; + p2 = knots[i+2]; + p3 = knots[i+3]; + + return (stbir_real) 0.5*((stbir_real) 2.0*p1 + t*((p2 - p0) + t*(((stbir_real) 2.0*p0 - (stbir_real) 5.0*p1 + (stbir_real) 4.0*p2 - p3) + t*(((stbir_real) 3.0*p1 - (stbir_real) 3.0*p2) + (p3 - p0))))); +} + +// point sampling takes the one nearest input pixel in both directions. +static stbir_real stbir__radius_point(stbir_real scale, stbir_real inv_scale, int is_upsample, void * user_data) +{ + STBIR__UNUSED(scale); + STBIR__UNUSED(inv_scale); + STBIR__UNUSED(is_upsample); + STBIR__UNUSED(user_data); + return (stbir_real)0.5; +} + +// the other built-ins' supports: the kernel's radius in filter pixels + +static stbir_real stbir__support_one(stbir_real scale, void * user_data) +{ + STBIR__UNUSED(scale); + STBIR__UNUSED(user_data); + return (stbir_real)1.0; +} + +static stbir_real stbir__support_two(stbir_real scale, void * user_data) +{ + STBIR__UNUSED(scale); + STBIR__UNUSED(user_data); + return (stbir_real)2.0; +} + +static stbir_real stbir__support_two_and_half(stbir_real scale, void * user_data) +{ + STBIR__UNUSED(scale); + STBIR__UNUSED(user_data); + return (stbir_real)2.5; +} + +static stbir_real stbir__support_three(stbir_real scale, void * user_data) +{ + STBIR__UNUSED(scale); + STBIR__UNUSED(user_data); + return (stbir_real)3.0; +} + +// the built-in filters, indexed by stbir_filter (everything below STBIR_FILTER_OTHER +static stbir__kernel_callback_new * stbir__builtin_kernels[STBIR_FILTER_OTHER] = +{ + 0, // STBIR_FILTER_DEFAULT (resolved before the lookup) + stbir__filter_trapezoid, // STBIR_FILTER_BOX + stbir__filter_triangle, // STBIR_FILTER_TRIANGLE + stbir__filter_cubic, // STBIR_FILTER_CUBICBSPLINE + stbir__filter_catmullrom, // STBIR_FILTER_CATMULLROM + stbir__filter_mitchell, // STBIR_FILTER_MITCHELL + stbir__filter_point, // STBIR_FILTER_POINT_SAMPLE + stbir__filter_mitchell_quarter, // STBIR_FILTER_MITCHELL_QUARTER + stbir__filter_mitchell_sixth, // STBIR_FILTER_MITCHELL_SIXTH + stbir__filter_mitchell_neg_sixth, // STBIR_FILTER_MITCHELL_NEG_SIXTH + stbir__filter_mitchell_neg_third, // STBIR_FILTER_MITCHELL_NEG_THIRD + stbir__filter_lanczos4, // STBIR_FILTER_LANCZOS4 + stbir__filter_lanczos5, // STBIR_FILTER_LANCZOS5 + stbir__filter_muratori6, // STBIR_FILTER_MURATORI6 +}; + +static stbir__support_callback_new * stbir__builtin_supports[STBIR_FILTER_OTHER] = +{ + 0, // STBIR_FILTER_DEFAULT + 0, // STBIR_FILTER_BOX (a radius) + stbir__support_one, // STBIR_FILTER_TRIANGLE + stbir__support_two, // STBIR_FILTER_CUBICBSPLINE + stbir__support_two, // STBIR_FILTER_CATMULLROM + stbir__support_two, // STBIR_FILTER_MITCHELL + 0, // STBIR_FILTER_POINT_SAMPLE (a radius) + stbir__support_two, // STBIR_FILTER_MITCHELL_QUARTER + stbir__support_two, // STBIR_FILTER_MITCHELL_SIXTH + stbir__support_two, // STBIR_FILTER_MITCHELL_NEG_SIXTH + stbir__support_two, // STBIR_FILTER_MITCHELL_NEG_THIRD + stbir__support_two, // STBIR_FILTER_LANCZOS4 + stbir__support_two_and_half, // STBIR_FILTER_LANCZOS5 + stbir__support_three, // STBIR_FILTER_MURATORI6 +}; + +static stbir__radius_callback_new * stbir__builtin_radii[STBIR_FILTER_OTHER] = +{ + 0, // STBIR_FILTER_DEFAULT + stbir__radius_trapezoid, // STBIR_FILTER_BOX + 0, 0, 0, 0, // STBIR_FILTER_TRIANGLE .. STBIR_FILTER_MITCHELL (supports) + stbir__radius_point, // STBIR_FILTER_POINT_SAMPLE + 0, 0, 0, 0, 0, 0, 0, // STBIR_FILTER_MITCHELL_QUARTER .. STBIR_FILTER_MURATORI6 (supports) +}; + +// upsample (1:1 included), and a 1:1 with the output on the input's pixel grid (a copy) +static int stbir__scale_is_upsample( stbir__scale_info const * scale_info ) +{ + return ( scale_info->scale >= (stbir_real) 1.0 ); +} + +static int stbir__scale_is_copy( stbir__scale_info const * scale_info ) +{ + return ( scale_info->positions_exact ) && ( scale_info->scale_numerator == scale_info->scale_denominator ); +} + +// the sum under which a coefficient set is zeroed +static stbir_real stbir__too_small_coefficient_sum_for( stbir__scale_info const * scale_info ) +{ + if ( stbir__scale_is_upsample( scale_info ) ) + return stbir__too_small_coefficient_sum; + return stbir__too_small_coefficient_sum * scale_info->scale; +} + +// floor and ceil without math.h (the header doesn't use it), either sign, for pixel coordinates +// and tap counts - the range is known to fit an int +static int stbir__floor_to_int( stbir_real x ) +{ + int i = (int) x; + return ( (stbir_real)i > x ) ? i - 1 : i; +} + +static int stbir__ceil_to_int( stbir_real x ) +{ + int i = (int) x; + return ( (stbir_real)i < x ) ? i + 1 : i; +} + +// the filter's radius in input pixels. a radius callback gives it directly. a support callback gives the radius in +// filter pixels (called with the ratio <= 1): input pixels when upsampling, output pixels when downsampling, so it +// is scaled by inv_scale there. the old float form is converted the same way. +static stbir_real stbir__filter_radius_in_input_pixels( stbir__radius_callback_new * radius, stbir__support_callback_new * support_new, stbir__support_callback * support_old, stbir__scale_info const * scale_info, void * user_data ) +{ + int is_upsample = stbir__scale_is_upsample( scale_info ); + stbir_real r; + + if ( radius ) + r = radius( scale_info->scale, scale_info->inv_scale, is_upsample, user_data ); + else if ( is_upsample ) + { + if ( support_new ) + r = support_new( scale_info->inv_scale, user_data ); + else + r = (stbir_real) support_old( (float) scale_info->inv_scale, user_data ); + } + else + { + if ( support_new ) + r = support_new( scale_info->scale, user_data ); + else + r = (stbir_real) support_old( (float) scale_info->scale, user_data ); + r *= scale_info->inv_scale; + } + + // at least half an input pixel (point sampling's), so every output has a tap: the downsample generator walks the + // inputs and gives each output the ones within reach. a user filter's might be under; the built-ins never are + if ( !( r >= (stbir_real)0.5 ) ) + r = (stbir_real)0.5; + + return r; +} + +static stbir_real stbir__filter_radius_in_output_pixels( stbir__radius_callback_new * radius, stbir__support_callback_new * support_new, stbir__support_callback * support_old, stbir__scale_info const * scale_info, void * user_data ) +{ + return stbir__filter_radius_in_input_pixels( radius, support_new, support_old, scale_info, user_data ) / scale_info->inv_scale; } // This is the maximum number of input samples that can affect an output sample // with the given filter from the output pixel's perspective -static int stbir__get_filter_pixel_width(stbir__support_callback * support, float scale, void * user_data) +static int stbir__get_filter_pixel_width( stbir__radius_callback_new * radius, stbir__support_callback_new * support_new, stbir__support_callback * support_old, stbir__scale_info const * scale_info, void * user_data) { - STBIR_ASSERT(support != 0); - - if ( scale >= ( 1.0f-stbir__small_float ) ) // upscale - return (int)STBIR_CEILF(support(1.0f/scale,user_data) * 2.0f); - else - return (int)STBIR_CEILF(support(scale,user_data) * 2.0f / scale); + STBIR_ASSERT( ( radius != 0 ) || ( support_new != 0 ) || ( support_old != 0 ) ); + return stbir__ceil_to_int( (stbir_real)2.0 * stbir__filter_radius_in_input_pixels( radius, support_new, support_old, scale_info, user_data ) ); } -// this is how many coefficents per run of the filter (which is different -// from the filter_pixel_width depending on if we are scattering or gathering) +// this is how many coefficients per run of the filter (which is different from the filter_pixel_width depending on if we are scattering or gathering) static int stbir__get_coefficient_width(stbir__sampler * samp, int is_gather, void * user_data) { - float scale = samp->scale_info.scale; - stbir__support_callback * support = samp->filter_support; + stbir__scale_info const * scale_info = &samp->scale_info; switch( is_gather ) { case 1: - return (int)STBIR_CEILF(support(1.0f / scale, user_data) * 2.0f); case 2: - return (int)STBIR_CEILF(support(scale, user_data) * 2.0f / scale); + // one of slack: a tap exactly on the far edge is in the range (stbir__exact_pixel_range), one more than ceil( 2 x radius ) + return stbir__ceil_to_int( (stbir_real)2.0 * stbir__filter_radius_in_input_pixels( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, scale_info, user_data ) ) + 1; case 0: - return (int)STBIR_CEILF(support(scale, user_data) * 2.0f); + return stbir__ceil_to_int( (stbir_real)2.0 * stbir__filter_radius_in_output_pixels( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, scale_info, user_data ) ); default: STBIR_ASSERT( (is_gather >= 0 ) && (is_gather <= 2 ) ); return 0; @@ -3073,11 +3493,10 @@ static int stbir__edge_reflect_full( int n, int max ) if (n >= max) { - int max2 = max * 2; - if (n >= max2) + if ( ( n - max ) >= max ) // (as n >= 2 max, without the overflow) return 0; else - return max2 - n - 1; + return ( max - 1 ) - ( n - max ); } return n; // NOTREACHED @@ -3115,212 +3534,106 @@ stbir__inline static int stbir__edge_wrap(stbir_edge edge, int n, int max) return stbir__edge_wrap_slow[edge]( n, max ); } -#define STBIR__MERGE_RUNS_PIXEL_THRESHOLD 16 - -// get information on the extents of a sampler -static void stbir__get_extents( stbir__sampler * samp, stbir__extents * scanline_extents ) +// the ranges' edge handling: in pixel ranges may hang off the image with WRAP (the decoder wraps them, within a width +// each side), out pixel ranges are clipped to the output +static void stbir__clip_in_pixel_range( int first, int last, int input_size, stbir_edge edge, int * first_pixel, int * last_pixel ) { - int j, stop; - int left_margin, right_margin; - int min_n = 0x7fffffff, max_n = -0x7fffffff; - int min_left = 0x7fffffff, max_left = -0x7fffffff; - int min_right = 0x7fffffff, max_right = -0x7fffffff; - stbir_edge edge = samp->edge; - stbir__contributors* contributors = samp->contributors; - int output_sub_size = samp->scale_info.output_sub_size; - int input_full_size = samp->scale_info.input_full_size; - int filter_pixel_margin = samp->filter_pixel_margin; - - STBIR_ASSERT( samp->is_gather ); - - stop = output_sub_size; - for (j = 0; j < stop; j++ ) - { - STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 ); - if ( contributors[j].n0 < min_n ) - { - min_n = contributors[j].n0; - stop = j + filter_pixel_margin; // if we find a new min, only scan another filter width - if ( stop > output_sub_size ) stop = output_sub_size; - } - } - - stop = 0; - for (j = output_sub_size - 1; j >= stop; j-- ) - { - STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 ); - if ( contributors[j].n1 > max_n ) - { - max_n = contributors[j].n1; - stop = j - filter_pixel_margin; // if we find a new max, only scan another filter width - if (stop<0) stop = 0; - } - } - - STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n ); - STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n ); - - // now calculate how much into the margins we really read - left_margin = 0; - if ( min_n < 0 ) - { - left_margin = -min_n; - min_n = 0; - } - - right_margin = 0; - if ( max_n >= input_full_size ) - { - right_margin = max_n - input_full_size + 1; - max_n = input_full_size - 1; - } - - // index 1 is margin pixel extents (how many pixels we hang over the edge) - scanline_extents->edge_sizes[0] = left_margin; - scanline_extents->edge_sizes[1] = right_margin; - - // index 2 is pixels read from the input - scanline_extents->spans[0].n0 = min_n; - scanline_extents->spans[0].n1 = max_n; - scanline_extents->spans[0].pixel_offset_for_input = min_n; - - // default to no other input range - scanline_extents->spans[1].n0 = 0; - scanline_extents->spans[1].n1 = -1; - scanline_extents->spans[1].pixel_offset_for_input = 0; - - // don't have to do edge calc for zero clamp - if ( edge == STBIR_EDGE_ZERO ) - return; - - // convert margin pixels to the pixels within the input (min and max) - for( j = -left_margin ; j < 0 ; j++ ) - { - int p = stbir__edge_wrap( edge, j, input_full_size ); - if ( p < min_left ) - min_left = p; - if ( p > max_left ) - max_left = p; - } - - for( j = input_full_size ; j < (input_full_size + right_margin) ; j++ ) - { - int p = stbir__edge_wrap( edge, j, input_full_size ); - if ( p < min_right ) - min_right = p; - if ( p > max_right ) - max_right = p; - } - - // merge the left margin pixel region if it connects within 4 pixels of main pixel region - if ( min_left != 0x7fffffff ) - { - if ( ( ( min_left <= min_n ) && ( ( max_left + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) || - ( ( min_n <= min_left ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_left ) ) ) - { - scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_left ); - scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_left ); - scanline_extents->spans[0].pixel_offset_for_input = min_n; - left_margin = 0; - } - } - - // merge the right margin pixel region if it connects within 4 pixels of main pixel region - if ( min_right != 0x7fffffff ) - { - if ( ( ( min_right <= min_n ) && ( ( max_right + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) || - ( ( min_n <= min_right ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_right ) ) ) - { - scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_right ); - scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_right ); - scanline_extents->spans[0].pixel_offset_for_input = min_n; - right_margin = 0; - } - } - - STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n ); - STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n ); - - // you get two ranges when you have the WRAP edge mode and you are doing just the a piece of the resize - // so you need to get a second run of pixels from the opposite side of the scanline (which you - // wouldn't need except for WRAP) - - - // if we can't merge the min_left range, add it as a second range - if ( ( left_margin ) && ( min_left != 0x7fffffff ) ) - { - stbir__span * newspan = scanline_extents->spans + 1; - STBIR_ASSERT( right_margin == 0 ); - if ( min_left < scanline_extents->spans[0].n0 ) - { - scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0; - scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0; - scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1; - --newspan; - } - newspan->pixel_offset_for_input = min_left; - newspan->n0 = -left_margin; - newspan->n1 = ( max_left - min_left ) - left_margin; - scanline_extents->edge_sizes[0] = 0; // don't need to copy the left margin, since we are directly decoding into the margin - } - // if we can't merge the min_right range, add it as a second range - else - if ( ( right_margin ) && ( min_right != 0x7fffffff ) ) - { - stbir__span * newspan = scanline_extents->spans + 1; - if ( min_right < scanline_extents->spans[0].n0 ) - { - scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0; - scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0; - scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1; - --newspan; - } - newspan->pixel_offset_for_input = min_right; - newspan->n0 = scanline_extents->spans[1].n1 + 1; - newspan->n1 = scanline_extents->spans[1].n1 + 1 + ( max_right - min_right ); - scanline_extents->edge_sizes[1] = 0; // don't need to copy the right margin, since we are directly decoding into the margin - } - - // sort the spans into write output order - if ( ( scanline_extents->spans[1].n1 > scanline_extents->spans[1].n0 ) && ( scanline_extents->spans[0].n0 > scanline_extents->spans[1].n0 ) ) - { - stbir__span tspan = scanline_extents->spans[0]; - scanline_extents->spans[0] = scanline_extents->spans[1]; - scanline_extents->spans[1] = tspan; - } -} - -static void stbir__calculate_in_pixel_range( int * first_pixel, int * last_pixel, float out_pixel_center, float out_filter_radius, float inv_scale, float out_shift, int input_size, stbir_edge edge ) -{ - int first, last; - float out_pixel_influence_lowerbound = out_pixel_center - out_filter_radius; - float out_pixel_influence_upperbound = out_pixel_center + out_filter_radius; - - float in_pixel_influence_lowerbound = (out_pixel_influence_lowerbound + out_shift) * inv_scale; - float in_pixel_influence_upperbound = (out_pixel_influence_upperbound + out_shift) * inv_scale; - - first = (int)(STBIR_FLOORF(in_pixel_influence_lowerbound + 0.5f)); - last = (int)(STBIR_FLOORF(in_pixel_influence_upperbound - 0.5f)); - if ( last < first ) last = first; // point sample mode can span a value *right* at 0.5, and cause these to cross - + if ( last < first ) + last = first; // point sample mode can span a value *right* at 0.5, and cause these to cross if ( edge == STBIR_EDGE_WRAP ) { if ( first < -input_size ) first = -input_size; - if ( last >= (input_size*2)) - last = (input_size*2) - 1; + if ( ( last - input_size ) >= input_size ) // (as last >= 2 input_size, without the overflow) + last = ( input_size - 1 ) + input_size; } - *first_pixel = first; *last_pixel = last; } -static void stbir__calculate_coefficients_for_gather_upsample( float out_filter_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float* coefficient_group, int coefficient_width, stbir_edge edge, void * user_data ) +static void stbir__clip_out_pixel_range( int first, int last, int out_size, int * first_pixel, int * last_pixel ) +{ + if ( first < 0 ) + first = 0; + if ( last >= out_size ) + last = out_size - 1; + *first_pixel = first; + *last_pixel = last; +} + +// The pixels within support of the center numer / den (quotient kept apart, only the remainder meets the real math). +// On the far edge is in, on the near edge out; stbir__tie_epsilon lifts the tie above rounding noise (point sampling +// hits both edges at every even ratio). The output range and the real chain ranges below use the same rule. +#define stbir__tie_epsilon ( ( sizeof( stbir_real ) == 4 ) ? (stbir_real)( 1.0 / ( 1 << 18 ) ) : (stbir_real)( 1.0 / ( (stbir_int64)1 << 36 ) ) ) + +// The input pixels within out_filter_radius (output pixels) of an output pixel's center. +static void stbir__calculate_in_pixel_range( int * first_pixel, int * last_pixel, stbir_real out_pixel_center, stbir_real out_filter_radius, stbir_real inv_scale, stbir_real out_shift, int input_size, stbir_edge edge ) +{ + stbir_real in_pixel_influence_lowerbound = ( out_pixel_center - out_filter_radius + out_shift ) * inv_scale; + stbir_real in_pixel_influence_upperbound = ( out_pixel_center + out_filter_radius + out_shift ) * inv_scale; + stbir__clip_in_pixel_range( stbir__floor_to_int( in_pixel_influence_lowerbound + (stbir_real)0.5 + stbir__tie_epsilon ), stbir__floor_to_int( in_pixel_influence_upperbound - (stbir_real)0.5 + stbir__tie_epsilon ), input_size, edge, first_pixel, last_pixel ); +} + +// Output pixel o's center against input pixel i's (in input pixels for upsamples, output pixels for downsamples). With an +// exact scale N/D and integer shift it is ( ( 2o + 1 + 2 shift ) D - ( 2i + 1 ) N ) / 2N (or 2D): integers until one +// multiply by the precomputed 1 / 2N (or 1 / 2D), exact to float at any coordinate. +static stbir_real stbir__tap_offset( stbir__scale_info const * scale_info, int o, int i, int in_input_pixels ) +{ + if ( scale_info->positions_exact ) + { + // (the difference is at most the filter's width times 2N or 2D: exact in a double, a float ulp in a float) + stbir_int64 numer = ( 2 * (stbir_int64)o + 1 + 2 * (stbir_int64)scale_info->shift_pixels ) * (stbir_int64)scale_info->scale_denominator + - ( 2 * (stbir_int64)i + 1 ) * (stbir_int64)scale_info->scale_numerator; + return (stbir_real)numer * ( in_input_pixels ? scale_info->tap_scale_in : scale_info->tap_scale_out ); + } + if ( in_input_pixels ) + return ( ( (stbir_real)o + (stbir_real)0.5 + scale_info->pixel_shift ) * scale_info->inv_scale ) - ( (stbir_real)i + (stbir_real)0.5 ); + return ( (stbir_real)o + (stbir_real)0.5 ) - ( ( (stbir_real)i + (stbir_real)0.5 ) * scale_info->scale - scale_info->pixel_shift ); +} + +// the kernel's weight at x (in filter pixels): through the new form directly, or an old float kernel's floats +static stbir_real stbir__kernel_weight( stbir__kernel_callback_new * kernel_new, stbir__kernel_callback * kernel_old, stbir_real x, stbir_real scale, void * user_data ) +{ + if ( kernel_new ) + return kernel_new( x, scale, user_data ); + return (stbir_real) kernel_old( (float) x, (float) scale, user_data ); +} + +static void stbir__exact_pixel_range( stbir_int64 numer, stbir_int64 den, stbir_real den_reciprocal, stbir_real support, int * first, int * last ) +{ + int q = (int)( numer / den ); + stbir_real f = (stbir_real)( numer % den ) * den_reciprocal; // ( -1, 1 ): the same sign as numer + *first = q + stbir__floor_to_int( f - support + (stbir_real)0.5 + stbir__tie_epsilon ); + *last = q + stbir__floor_to_int( f + support - (stbir_real)0.5 + stbir__tie_epsilon ); +} + +static void stbir__exact_output_range( stbir_int64 numer, stbir_int64 den, stbir_real den_reciprocal, stbir_real support, int * first, int * last ) +{ + int q = (int)( numer / den ); + stbir_real f = (stbir_real)( numer % den ) * den_reciprocal; + *first = q + stbir__floor_to_int( f - support - (stbir_real)0.5 - stbir__tie_epsilon ) + 1; // ceil, a tie exactly on it included + *last = q + stbir__floor_to_int( f + support - (stbir_real)0.5 - stbir__tie_epsilon ); // a tie exactly on it excluded +} + +// The input pixels within the filter's support of output pixel o: out_filter_radius is the support in output pixels for +// the real chain, support the same in input pixels (the filter's own, for upsamples) for the exact positions - then +// the center is the one stbir__tap_offset weights from. +static void stbir__output_in_pixel_range( stbir__scale_info const * scale_info, int o, stbir_real out_filter_radius, stbir_real support, stbir_edge edge, int * first_pixel, int * last_pixel ) +{ + if ( scale_info->positions_exact ) + { + stbir_int64 numer = ( 2 * (stbir_int64)o + 1 + 2 * (stbir_int64)scale_info->shift_pixels ) * (stbir_int64)scale_info->scale_denominator; // output o's center in input pixels, over 2N + int first, last; + stbir__exact_pixel_range( numer, 2 * (stbir_int64)scale_info->scale_numerator, scale_info->tap_scale_in, support, &first, &last ); + stbir__clip_in_pixel_range( first, last, scale_info->input_full_size, edge, first_pixel, last_pixel ); + } + else + stbir__calculate_in_pixel_range( first_pixel, last_pixel, (stbir_real)o + (stbir_real)0.5, out_filter_radius, scale_info->inv_scale, scale_info->pixel_shift, scale_info->input_full_size, edge ); +} + +static void stbir__calculate_coefficients_for_gather_upsample( stbir_real out_filter_radius, stbir_real support, stbir__kernel_callback_new * kernel_new, stbir__kernel_callback * kernel_old, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float* coefficient_group, int coefficient_width, stbir_edge edge, void * user_data ) { int n, end; - float inv_scale = scale_info->inv_scale; - float out_shift = scale_info->pixel_shift; - int input_size = scale_info->input_full_size; int numerator = scale_info->scale_numerator; int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) ); @@ -3330,12 +3643,9 @@ static void stbir__calculate_coefficients_for_gather_upsample( float out_filter_ { int i; int last_non_zero; - float out_pixel_center = (float)n + 0.5f; - float in_center_of_out = (out_pixel_center + out_shift) * inv_scale; - int in_first_pixel, in_last_pixel; - stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, out_pixel_center, out_filter_radius, inv_scale, out_shift, input_size, edge ); + stbir__output_in_pixel_range( scale_info, n, out_filter_radius, support, edge, &in_first_pixel, &in_last_pixel ); // make sure we never generate a range larger than our precalculated coeff width // this only happens in point sample mode, but it's a good safe thing to do anyway @@ -3345,11 +3655,10 @@ static void stbir__calculate_coefficients_for_gather_upsample( float out_filter_ last_non_zero = -1; for (i = 0; i <= in_last_pixel - in_first_pixel; i++) { - float in_pixel_center = (float)(i + in_first_pixel) + 0.5f; - float coeff = kernel(in_center_of_out - in_pixel_center, inv_scale, user_data); + stbir_real weight = stbir__kernel_weight( kernel_new, kernel_old, stbir__tap_offset( scale_info, n, i + in_first_pixel, 1 ), scale_info->inv_scale, user_data ); + float coeff = (float) weight; - // kill denormals - if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) ) + if ( ( weight < stbir__too_small_coefficient ) && ( weight > -stbir__too_small_coefficient ) ) { if ( i == 0 ) // if we're at the front, just eat zero contributors { @@ -3419,30 +3728,35 @@ static void stbir__insert_coeff( stbir__contributors * contribs, float * coeffs, } } -static void stbir__calculate_out_pixel_range( int * first_pixel, int * last_pixel, float in_pixel_center, float in_pixels_radius, float scale, float out_shift, int out_size ) +// The output pixels within in_pixels_radius (input pixels) of an input pixel's center. +static void stbir__calculate_out_pixel_range( int * first_pixel, int * last_pixel, stbir_real in_pixel_center, stbir_real in_pixels_radius, stbir_real scale, stbir_real out_shift, int out_size ) { - float in_pixel_influence_lowerbound = in_pixel_center - in_pixels_radius; - float in_pixel_influence_upperbound = in_pixel_center + in_pixels_radius; - float out_pixel_influence_lowerbound = in_pixel_influence_lowerbound * scale - out_shift; - float out_pixel_influence_upperbound = in_pixel_influence_upperbound * scale - out_shift; - int out_first_pixel = (int)(STBIR_FLOORF(out_pixel_influence_lowerbound + 0.5f)); - int out_last_pixel = (int)(STBIR_FLOORF(out_pixel_influence_upperbound - 0.5f)); - - if ( out_first_pixel < 0 ) - out_first_pixel = 0; - if ( out_last_pixel >= out_size ) - out_last_pixel = out_size - 1; - *first_pixel = out_first_pixel; - *last_pixel = out_last_pixel; + stbir_real out_pixel_influence_lowerbound = ( in_pixel_center - in_pixels_radius ) * scale - out_shift; + stbir_real out_pixel_influence_upperbound = ( in_pixel_center + in_pixels_radius ) * scale - out_shift; + stbir__clip_out_pixel_range( stbir__floor_to_int( out_pixel_influence_lowerbound - (stbir_real)0.5 - stbir__tie_epsilon ) + 1, stbir__floor_to_int( out_pixel_influence_upperbound - (stbir_real)0.5 - stbir__tie_epsilon ), out_size, first_pixel, last_pixel ); } -static void stbir__calculate_coefficients_for_gather_downsample( int start, int end, float in_pixels_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int coefficient_width, int num_contributors, stbir__contributors * contributors, float * coefficient_group, void * user_data ) +// The output pixels within the filter's support of input pixel i, one pixel at a time - the same ways as +// stbir__output_in_pixel_range: in_pixels_radius is the support in input pixels for the real chain, support the +// same in output pixels for the exact positions. +static void stbir__input_out_pixel_range( stbir__scale_info const * scale_info, int i, stbir_real in_pixels_radius, stbir_real support, int * first_pixel, int * last_pixel ) +{ + if ( scale_info->positions_exact ) + { + stbir_int64 numer = ( 2 * (stbir_int64)i + 1 ) * (stbir_int64)scale_info->scale_numerator - 2 * (stbir_int64)scale_info->shift_pixels * (stbir_int64)scale_info->scale_denominator; // input i's center in output pixels, over 2D + int first, last; + stbir__exact_output_range( numer, 2 * (stbir_int64)scale_info->scale_denominator, scale_info->tap_scale_out, support, &first, &last ); + stbir__clip_out_pixel_range( first, last, scale_info->output_sub_size, first_pixel, last_pixel ); + } + else + stbir__calculate_out_pixel_range( first_pixel, last_pixel, (stbir_real)i + (stbir_real)0.5, in_pixels_radius, scale_info->scale, scale_info->pixel_shift, scale_info->output_sub_size ); +} + +static void stbir__calculate_coefficients_for_gather_downsample( int start, int end, stbir_real in_pixels_radius, stbir_real support, stbir__kernel_callback_new * kernel_new, stbir__kernel_callback * kernel_old, stbir__scale_info * scale_info, int coefficient_width, int num_contributors, stbir__contributors * contributors, float * coefficient_group, void * user_data ) { int in_pixel; int i; int first_out_inited = -1; - float scale = scale_info->scale; - float out_shift = scale_info->pixel_shift; int out_size = scale_info->output_sub_size; int numerator = scale_info->scale_numerator; int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < out_size ) ); @@ -3452,11 +3766,9 @@ static void stbir__calculate_coefficients_for_gather_downsample( int start, int // Loop through the input pixels for (in_pixel = start; in_pixel < end; in_pixel++) { - float in_pixel_center = (float)in_pixel + 0.5f; - float out_center_of_in = in_pixel_center * scale - out_shift; int out_first_pixel, out_last_pixel; - stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, in_pixel_center, in_pixels_radius, scale, out_shift, out_size ); + stbir__input_out_pixel_range( scale_info, in_pixel, in_pixels_radius, support, &out_first_pixel, &out_last_pixel ); if ( out_first_pixel > out_last_pixel ) continue; @@ -3465,7 +3777,7 @@ static void stbir__calculate_coefficients_for_gather_downsample( int start, int if ( polyphase ) { // when polyphase, you only have to do coeffs up to the numerator count - if ( out_first_pixel == numerator ) + if ( out_first_pixel >= numerator ) break; // don't do any extra work, clamp last pixel at numerator too @@ -3475,17 +3787,15 @@ static void stbir__calculate_coefficients_for_gather_downsample( int start, int for (i = 0; i <= out_last_pixel - out_first_pixel; i++) { - float out_pixel_center = (float)(i + out_first_pixel) + 0.5f; - float x = out_pixel_center - out_center_of_in; - float coeff = kernel(x, scale, user_data) * scale; + stbir_real weight = stbir__kernel_weight( kernel_new, kernel_old, stbir__tap_offset( scale_info, i + out_first_pixel, in_pixel, 0 ), scale_info->scale, user_data ); + float coeff = 0.0f; - // kill the coeff if it's too small (avoid denormals) - if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) ) - coeff = 0.0f; + if ( ( weight >= stbir__too_small_coefficient ) || ( weight <= -stbir__too_small_coefficient ) ) + coeff = (float) ( weight * scale_info->scale ); { int out = i + out_first_pixel; - float * coeffs = coefficient_group + out * coefficient_width; + float * coeffs = coefficient_group + (size_t)out * coefficient_width; stbir__contributors * contribs = contributors + out; // is this the first time this output pixel has been seen? Init it. @@ -3500,7 +3810,7 @@ static void stbir__calculate_coefficients_for_gather_downsample( int start, int else { // insert on end (always in order) - if ( coeffs[0] == 0.0f ) // if the first coefficent is zero, then zap it for this coeffs + if ( coeffs[0] == 0.0f ) // if the first coefficient is zero, then zap it for this coeff { STBIR_ASSERT( ( in_pixel - contribs->n0 ) == 1 ); // ensure that when we zap, we're at the 2nd pos contribs->n0 = in_pixel; @@ -3514,351 +3824,216 @@ static void stbir__calculate_coefficients_for_gather_downsample( int start, int } } -#ifdef STBIR_RENORMALIZE_IN_FLOAT -#define STBIR_RENORM_TYPE float -#else -#define STBIR_RENORM_TYPE double -#endif +// fold a contributor's out of range taps back into the image for the edge mode, trim its trailing zero weights, +// and fold its extents into lowest/highest/widest +static stbir__inline void stbir__fold_and_trim_contributor( stbir_edge edge, stbir__contributors * contributor, float * coefficient_group, int coefficient_width, int input_size, stbir__filter_extent_info * extents ) +{ + int input_last_pixel = input_size - 1; + int i; -static void stbir__cleanup_gathered_coefficients( stbir_edge edge, stbir__filter_extent_info* filter_info, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float * coefficient_group, int coefficient_width ) + if ( edge == STBIR_EDGE_ZERO ) + { + // In zero edge mode, just remove the out of bounds pixels completely (their weights are accounted for now). + if ( contributor->n1 > input_last_pixel ) + contributor->n1 = input_last_pixel; + + if ( contributor->n0 < 0 ) + { + int skip = -contributor->n0; + int remaining; + + contributor->n0 = 0; + + // Move the weights down. + remaining = contributor->n1 - contributor->n0 + 1; + for (i = 0; i < remaining; i++) + coefficient_group[i] = coefficient_group[i + skip]; + } + } + else if ( ( edge == STBIR_EDGE_CLAMP ) || ( edge == STBIR_EDGE_REFLECT ) ) + { + // For clamp and reflect, calculate the true in bounds pixel (based on the edge type) and add the weight to it. + + // Right hand side first. + if ( contributor->n1 > input_last_pixel ) + { + int first = contributor->n0; + int last = contributor->n1; + contributor->n1 = input_last_pixel; + for (i = input_size; i <= last; i++) + stbir__insert_coeff( contributor, coefficient_group, stbir__edge_wrap_slow[edge]( i, input_size ), coefficient_group[i - first], coefficient_width ); + } + + // Then the left. + if ( contributor->n0 < 0 ) + { + int first = contributor->n0; + float * coeff = coefficient_group - ( first + 1 ); // the weight of pixel -1 + float first_coeff; + + // Reinsert the weights at their reflected or clamped pixel (insert accumulates if the pixel is already there). + for (i = -1; i > first; i--) + stbir__insert_coeff( contributor, coefficient_group, stbir__edge_wrap_slow[edge]( i, input_size ), *coeff--, coefficient_width ); + + // Save the first one, since we didn't do it yet (i == n0) - there might be too many coeffs to hold before we shrink. + first_coeff = coeff[0]; + + // Slide all the coeffs down (we accumulated them into the positive pixels) and reset the first pixel. + contributor->n0 = 0; + for (i = 0; i <= contributor->n1; i++) + coefficient_group[i] = coefficient_group[i - first]; + + // Now that we have shrunk the contributor, we can insert the first one safely. + stbir__insert_coeff( contributor, coefficient_group, stbir__edge_wrap_slow[edge]( first, input_size ), first_coeff, coefficient_width ); + } + } + + // Trim trailing zero coefficients, re-zero the unused tail, and widen the extents to this contributor. + if ( contributor->n0 <= contributor->n1 ) + { + int count = contributor->n1 - contributor->n0 + 1; + while ( count && ( coefficient_group[count - 1] == 0.0f ) ) + --count; + + contributor->n1 = contributor->n0 + count - 1; + + if ( count > 0 ) + { + if ( contributor->n0 < extents->lowest ) + extents->lowest = contributor->n0; + if ( contributor->n1 > extents->highest ) + extents->highest = contributor->n1; + if ( count > extents->widest ) + extents->widest = count; + } + + for (i = count; i < coefficient_width; i++) + coefficient_group[i] = 0.0f; + } +} + +static void stbir__cleanup_gathered_coefficients( stbir_edge edge, stbir__filter_extent_info* filter_info, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float * coefficients, int coefficient_width ) { int input_size = scale_info->input_full_size; - int input_last_n1 = input_size - 1; - int n, end; - int lowest = 0x7fffffff; - int highest = -0x7fffffff; - int widest = -1; int numerator = scale_info->scale_numerator; int denominator = scale_info->scale_denominator; int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) ); - float * coeffs; - stbir__contributors * contribs; + int calculated = polyphase ? numerator : num_contributors; // when polyphase, only the first numerator outputs were calculated + stbir_real too_small_sum = stbir__too_small_coefficient_sum_for( scale_info ); + int n, i; - // weight all the coeffs for each sample - coeffs = coefficient_group; - contribs = contributors; - end = num_contributors; if ( polyphase ) end = numerator; - for (n = 0; n < end; n++) + // Make the sum of each output's coefficients 1. + for (n = 0; n < calculated; n++) { - int i; - STBIR_RENORM_TYPE filter_scale, total_filter = 0; - int e; + stbir__contributors * contributor = contributors + n; + float * coefficient_group = coefficients + (size_t)n * coefficient_width; + stbir_real total_filter = 0; + int last = contributor->n1 - contributor->n0; - // add all contribs - e = contribs->n1 - contribs->n0; - for( i = 0 ; i <= e ; i++ ) + for (i = 0; i <= last; i++) { - total_filter += (STBIR_RENORM_TYPE) coeffs[i]; - STBIR_ASSERT( ( coeffs[i] >= -2.0f ) && ( coeffs[i] <= 2.0f ) ); // check for wonky weights + total_filter += (stbir_real) coefficient_group[i]; + STBIR_ASSERT( ( coefficient_group[i] >= -2.0f ) && ( coefficient_group[i] <= 2.0f ) ); // check for wonky weights } - // rescale - if ( ( total_filter < stbir__small_float ) && ( total_filter > -stbir__small_float ) ) + if ( ( total_filter < too_small_sum ) && ( total_filter > -too_small_sum ) ) { - // all coeffs are extremely small, just zero it - contribs->n1 = contribs->n0; - coeffs[0] = 0.0f; + // All coeffs are extremely small, just zero it. + contributor->n1 = contributor->n0; + coefficient_group[0] = 0.0f; + } + else if ( ( total_filter < ( (stbir_real)1.0 - stbir__renorm_tolerance ) ) || ( total_filter > ( (stbir_real)1.0 + stbir__renorm_tolerance ) ) ) + { + stbir_real filter_scale = (stbir_real)1.0 / total_filter; + for (i = 0; i <= last; i++) + coefficient_group[i] = (float) ( coefficient_group[i] * filter_scale ); + } + } + + // Polyphase: the rest of the outputs are the first numerator's, moved along by the denominator once per period. + if ( polyphase ) + { + for (n = numerator; n < num_contributors; n++) + { + contributors[n].n0 = contributors[n - numerator].n0 + denominator; + contributors[n].n1 = contributors[n - numerator].n1 + denominator; + } + stbir_overlapping_memcpy( coefficients + (size_t)numerator * coefficient_width, coefficients, (size_t)( num_contributors - numerator ) * coefficient_width ); + } + + filter_info->lowest = 0x7fffffff; + filter_info->highest = -0x7fffffff; + filter_info->widest = -1; + for (n = 0; n < num_contributors; n++) + stbir__fold_and_trim_contributor( edge, contributors + n, coefficients + (size_t)n * coefficient_width, coefficient_width, input_size, filter_info ); +} + +// how many input pixels the horizontal kernel that gets selected for this widest will read +static int stbir__horizontal_read_extent( stbir__contributors const * contribs, int widest ) +{ + int mod, extent; + + if ( widest <= 17 ) + return widest; + + mod = widest & 3; + extent = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod; + if ( extent < ( 8 + mod ) ) + extent = 8 + mod; + return extent; +} + +// the SIMD kernels load a remnant of three coefficients as four, so widths of 4n+3 get a fourth, zero coefficient +#define STBIR__COEFFICIENT_PAD( widest ) ( ( (widest) & 3 ) == 3 ) +#define STBIR__COLUMN_ENTRY_FLOATS( widest ) ( 2 + (widest) + STBIR__COEFFICIENT_PAD( widest ) ) // the entry's size in floats + +// Repack the coefficient groups from coefficient_width floats each to the run time layout: a stbir__column_entry, +// two ints (filled in by stbir__fill_column_entries) followed by widest coefficients. +static void stbir__pack_column_entries( int num_entries, float * coefficients, int coefficient_width, int widest ) +{ + int entry_floats = STBIR__COLUMN_ENTRY_FLOATS( widest ); + int i, j; + + for (i = 0; i < num_entries; i++) + { + // When the entries grow they move up, so do the last one first; otherwise they move down (or stay), first one first. + int n = ( entry_floats > coefficient_width ) ? ( num_entries - 1 - i ) : i; + float * coefficient_group = coefficients + (size_t)n * coefficient_width; + float * entry_coefficients = coefficients + (size_t)n * entry_floats + 2; + + if ( entry_coefficients > coefficient_group ) + { + for (j = widest - 1; j >= 0; j--) + entry_coefficients[j] = coefficient_group[j]; } else { - // if the total isn't 1.0, rescale everything - if ( ( total_filter < (1.0f-stbir__small_float) ) || ( total_filter > (1.0f+stbir__small_float) ) ) - { - filter_scale = ((STBIR_RENORM_TYPE)1.0) / total_filter; - - // scale them all - for (i = 0; i <= e; i++) - coeffs[i] = (float) ( coeffs[i] * filter_scale ); - } + for (j = 0; j < widest; j++) + entry_coefficients[j] = coefficient_group[j]; } - ++contribs; - coeffs += coefficient_width; + + // The kernels that load one float past the coefficients see a zero. + if ( STBIR__COEFFICIENT_PAD( widest ) ) + entry_coefficients[widest] = 0.0f; } - - // if we have a rational for the scale, we can exploit the polyphaseness to not calculate - // most of the coefficients, so we copy them here - if ( polyphase ) - { - stbir__contributors * prev_contribs = contributors; - stbir__contributors * cur_contribs = contributors + numerator; - - for( n = numerator ; n < num_contributors ; n++ ) - { - cur_contribs->n0 = prev_contribs->n0 + denominator; - cur_contribs->n1 = prev_contribs->n1 + denominator; - ++cur_contribs; - ++prev_contribs; - } - stbir_overlapping_memcpy( coefficient_group + numerator * coefficient_width, coefficient_group, ( num_contributors - numerator ) * coefficient_width * sizeof( coeffs[ 0 ] ) ); - } - - coeffs = coefficient_group; - contribs = contributors; - - for (n = 0; n < num_contributors; n++) - { - int i; - - // in zero edge mode, just remove out of bounds contribs completely (since their weights are accounted for now) - if ( edge == STBIR_EDGE_ZERO ) - { - // shrink the right side if necessary - if ( contribs->n1 > input_last_n1 ) - contribs->n1 = input_last_n1; - - // shrink the left side - if ( contribs->n0 < 0 ) - { - int j, left, skips = 0; - - skips = -contribs->n0; - contribs->n0 = 0; - - // now move down the weights - left = contribs->n1 - contribs->n0 + 1; - if ( left > 0 ) - { - for( j = 0 ; j < left ; j++ ) - coeffs[ j ] = coeffs[ j + skips ]; - } - } - } - else if ( ( edge == STBIR_EDGE_CLAMP ) || ( edge == STBIR_EDGE_REFLECT ) ) - { - // for clamp and reflect, calculate the true inbounds position (based on edge type) and just add that to the existing weight - - // right hand side first - if ( contribs->n1 > input_last_n1 ) - { - int start = contribs->n0; - int endi = contribs->n1; - contribs->n1 = input_last_n1; - for( i = input_size; i <= endi; i++ ) - stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), coeffs[i-start], coefficient_width ); - } - - // now check left hand edge - if ( contribs->n0 < 0 ) - { - int save_n0; - float save_n0_coeff; - float * c = coeffs - ( contribs->n0 + 1 ); - - // reinsert the coeffs with it reflected or clamped (insert accumulates, if the coeffs exist) - for( i = -1 ; i > contribs->n0 ; i-- ) - stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), *c--, coefficient_width ); - save_n0 = contribs->n0; - save_n0_coeff = c[0]; // save it, since we didn't do the final one (i==n0), because there might be too many coeffs to hold (before we resize)! - - // now slide all the coeffs down (since we have accumulated them in the positive contribs) and reset the first contrib - contribs->n0 = 0; - for(i = 0 ; i <= contribs->n1 ; i++ ) - coeffs[i] = coeffs[i-save_n0]; - - // now that we have shrunk down the contribs, we insert the first one safely - stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( save_n0, input_size ), save_n0_coeff, coefficient_width ); - } - } - - if ( contribs->n0 <= contribs->n1 ) - { - int diff = contribs->n1 - contribs->n0 + 1; - while ( diff && ( coeffs[ diff-1 ] == 0.0f ) ) - --diff; - - contribs->n1 = contribs->n0 + diff - 1; - - if ( contribs->n0 <= contribs->n1 ) - { - if ( contribs->n0 < lowest ) - lowest = contribs->n0; - if ( contribs->n1 > highest ) - highest = contribs->n1; - if ( diff > widest ) - widest = diff; - } - - // re-zero out unused coefficients (if any) - for( i = diff ; i < coefficient_width ; i++ ) - coeffs[i] = 0.0f; - } - - ++contribs; - coeffs += coefficient_width; - } - filter_info->lowest = lowest; - filter_info->highest = highest; - filter_info->widest = widest; } -#undef STBIR_RENORM_TYPE - -static int stbir__pack_coefficients( int num_contributors, stbir__contributors* contributors, float * coefficents, int coefficient_width, int widest, int row0, int row1 ) +// the horizontal kernels read a fixed number of pixels from each contributor's n0 (see stbir__horizontal_read_extent), +// so contributors near the right edge get moved back (zero weights in front) to keep those reads inside the +// decoded range. +static void stbir__clip_contributor_reads( int num_contributors, stbir__contributors* contributors, float * coefficients, int stride, int widest, int row0, int row1 ) { - #define STBIR_MOVE_1( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint32*)(dest))[0] = ((stbir_uint32*)(src))[0]; } - #define STBIR_MOVE_2( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; } - #ifdef STBIR_SIMD - #define STBIR_MOVE_4( dest, src ) { stbir__simdf t; STBIR_NO_UNROLL(dest); stbir__simdf_load( t, src ); stbir__simdf_store( dest, t ); } - #else - #define STBIR_MOVE_4( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; ((stbir_uint64*)(dest))[1] = ((stbir_uint64*)(src))[1]; } - #endif - int row_end = row1 + 1; STBIR__UNUSED( row0 ); // only used in an assert - if ( coefficient_width != widest ) - { - float * pc = coefficents; - float * coeffs = coefficents; - float * pc_end = coefficents + num_contributors * widest; - switch( widest ) - { - case 1: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_1( pc, coeffs ); - ++pc; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 2: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_2( pc, coeffs ); - pc += 2; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 3: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_2( pc, coeffs ); - STBIR_MOVE_1( pc+2, coeffs+2 ); - pc += 3; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 4: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - pc += 4; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 5: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_1( pc+4, coeffs+4 ); - pc += 5; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 6: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_2( pc+4, coeffs+4 ); - pc += 6; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 7: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_2( pc+4, coeffs+4 ); - STBIR_MOVE_1( pc+6, coeffs+6 ); - pc += 7; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 8: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_4( pc+4, coeffs+4 ); - pc += 8; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 9: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_4( pc+4, coeffs+4 ); - STBIR_MOVE_1( pc+8, coeffs+8 ); - pc += 9; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 10: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_4( pc+4, coeffs+4 ); - STBIR_MOVE_2( pc+8, coeffs+8 ); - pc += 10; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 11: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_4( pc+4, coeffs+4 ); - STBIR_MOVE_2( pc+8, coeffs+8 ); - STBIR_MOVE_1( pc+10, coeffs+10 ); - pc += 11; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - case 12: - STBIR_NO_UNROLL_LOOP_START - do { - STBIR_MOVE_4( pc, coeffs ); - STBIR_MOVE_4( pc+4, coeffs+4 ); - STBIR_MOVE_4( pc+8, coeffs+8 ); - pc += 12; - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - default: - STBIR_NO_UNROLL_LOOP_START - do { - float * copy_end = pc + widest - 4; - float * c = coeffs; - do { - STBIR_NO_UNROLL( pc ); - STBIR_MOVE_4( pc, c ); - pc += 4; - c += 4; - } while ( pc <= copy_end ); - copy_end += 4; - STBIR_NO_UNROLL_LOOP_START - while ( pc < copy_end ) - { - STBIR_MOVE_1( pc, c ); - ++pc; ++c; - } - coeffs += coefficient_width; - } while ( pc < pc_end ); - break; - } - } - - // some horizontal routines read one float off the end (which is then masked off), so put in a sentinel so we don't read an snan or denormal - coefficents[ widest * num_contributors ] = 8888.0f; - - // the minimum we might read for unrolled filters widths is 12. So, we need to - // make sure we never read outside the decode buffer, by possibly moving - // the sample area back into the scanline, and putting zeros weights first. + // the fixed width kernels (up to 17 taps) read exactly widest pixels from n0 whatever a contributor's own width is, + // and the generic ones read in fours (stbir__horizontal_read_extent). So we make sure we never read outside the + // decode buffer, by possibly moving the sample area back into the scanline, and putting zero weights first. // we start on the right edge and check until we're well past the possible // clip area (2*widest). { stbir__contributors * contribs = contributors + num_contributors - 1; - float * coeffs = coefficents + widest * ( num_contributors - 1 ); + float * coeffs = coefficients + (size_t)stride * ( num_contributors - 1 ); // go until no chance of clipping (this is usually less than 8 lops) while ( ( contribs >= contributors ) && ( ( contribs->n0 + widest*2 ) >= row_end ) ) @@ -3866,21 +4041,9 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* // might we clip?? if ( ( contribs->n0 + widest ) > row_end ) { - int stop_range = widest; - - // if range is larger than 12, it will be handled by generic loops that can terminate on the exact length - // of this contrib n1, instead of a fixed widest amount - so calculate this - if ( widest > 12 ) - { - int mod; - - // how far will be read in the n_coeff loop (which depends on the widest count mod4); - mod = widest & 3; - stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod; - - // the n_coeff loops do a minimum amount of coeffs, so factor that in! - if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod; - } + // past 17 taps the generic loops run, and they read to this contributor's own width rounded up to a four, + // not a fixed widest amount - so calculate this + int stop_range = stbir__horizontal_read_extent( contribs, widest ); // now see if we still clip with the refined range if ( ( contribs->n0 + stop_range ) > row_end ) @@ -3902,55 +4065,623 @@ static int stbir__pack_coefficients( int num_contributors, stbir__contributors* // zero new positions while ( to_co >= coeffs ) *to_co-- = 0; - // set new start point + // set new start point (backing up by less than the mod-4 slack keeps the read extent the same, so this can't clip again) contribs->n0 = new_n0; - if ( widest > 12 ) - { - int mod; - - // how far will be read in the n_coeff loop (which depends on the widest count mod4); - mod = widest & 3; - stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod; - - // the n_coeff loops do a minimum amount of coeffs, so factor that in! - if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod; - } + STBIR_ASSERT( ( contribs->n0 + stbir__horizontal_read_extent( contribs, widest ) ) <= row_end ); } } --contribs; - coeffs -= widest; + coeffs -= stride; + } + } +} + +// pack the coefficient sets into column entries and clip the edge reads +static int stbir__pack_coefficients( int num_contributors, stbir__contributors* contributors, float * coefficients, int coefficient_width, int widest, int row0, int row1 ) +{ + stbir__pack_column_entries( num_contributors, coefficients, coefficient_width, widest ); + stbir__clip_contributor_reads( num_contributors, contributors, coefficients + 2, STBIR__COLUMN_ENTRY_FLOATS( widest ), widest, row0, row1 ); + return widest; +} + +// For polyphase resizing (rational scale resizing), there is a ton of coefficent sharing across the +// horizontal scanline. We store the data like this (you need the pro and epi for clamp modes): +// [ prologue scanline coeffs ][ big run of repeating coeffs ][ epilogue scanline coeffs ] + +// Bounds on how many outputs at each edge have taps outside the image (before folding), from the filter geometry. +static void stbir__polyphase_edge_bounds( stbir__sampler const * samp, void * user_data, int * out_prologue, int * out_epilogue ) +{ + stbir__scale_info const * scale_info = &samp->scale_info; + int num_contributors = samp->num_contributors; + stbir_real in_pixels_radius; + int prologue = 0; + int epilogue = 0; + int n; + + // the reach of an output's taps, plus how far an approximated fraction's copies can drift from these real positions + in_pixels_radius = stbir__filter_radius_in_input_pixels( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, scale_info, user_data ) + (stbir_real)scale_info->approximated_drift; + + // Nothing gets folded with wrap. + if ( samp->edge != STBIR_EDGE_WRAP ) + { + // Looping through out pixels from the left edge, until one reads nothing before the image + for (n = 0; n < num_contributors; n++) + { + stbir_real in_first_pixel = ( (stbir_real)n + (stbir_real)0.5 + scale_info->pixel_shift ) * scale_info->inv_scale - in_pixels_radius; + if ( stbir__floor_to_int( in_first_pixel + (stbir_real)0.5 ) >= 0 ) + break; + ++prologue; + } + + // And from the right edge, until one reads nothing past it + for (n = num_contributors - 1; n >= 0; n--) + { + stbir_real in_last_pixel = ( (stbir_real)n + (stbir_real)0.5 + scale_info->pixel_shift ) * scale_info->inv_scale + in_pixels_radius; + if ( stbir__floor_to_int( in_last_pixel - (stbir_real)0.5 ) < scale_info->input_full_size ) + break; + ++epilogue; + } + + // A pixel of slack each way: this is the real chain, the generators' positions can be exact. + ++prologue; + ++epilogue; + } + + *out_prologue = prologue; + *out_epilogue = epilogue; +} + +// for polyphase (rational) scales, we want to unroll the sharing a bit just so we aren't +// constantly breaking out of a tight loop to back up the coeff sharing. +#define STBIR__POLYPHASE_MIN_BLOCK 256 + +// decide whether this (gather) sampler shares its polyphase coefficient sets, and size its tables accordingly. +// called before the allocation pass, so it can only use the sampler's geometry. +static void stbir__plan_polyphase_sharing( stbir__sampler * samp, int reads_get_clipped, void * user_data ) +{ + int num_contributors = samp->num_contributors; + int numerator = (int) samp->scale_info.scale_numerator; + int prologue, epilogue, period, repeating; + + samp->shared_period = 0; + samp->shared_prologue = 0; + samp->shared_epilogue = 0; + + if ( ( !samp->is_gather ) || ( !samp->scale_info.scale_is_rational ) || ( numerator >= num_contributors ) ) + return; + + stbir__polyphase_edge_bounds( samp, user_data, &prologue, &epilogue ); + + // the prologue is whole periods, so an output's phase is its index mod numerator + prologue = ( ( prologue + numerator - 1 ) / numerator ) * numerator; + + // the horizontal's clip (stbir__clip_contributor_reads) can also move outputs within 2*widest input pixels of the + // right edge: 2*widest*scale outputs, bounded with the coefficient width since widest isn't known yet + if ( reads_get_clipped ) + epilogue += (int)( (stbir_real)2.0 * (stbir_real)samp->coefficient_width * samp->scale_info.scale ) + 2; + + // the block is one period of sets (one set at period 1, run with an entry stride of zero). the horizontal walks a + // row through it, so its block is padded out to keep the kernel calls long; the vertical fetches a set per row by index + period = numerator; + if ( ( numerator > 1 ) && ( reads_get_clipped ) ) + { + while ( period < STBIR__POLYPHASE_MIN_BLOCK ) + period += numerator; + } + + // not enough repeating outputs to be worth it + repeating = num_contributors - prologue - epilogue; + if ( repeating < period ) + return; + + samp->shared_period = period; + samp->shared_prologue = prologue; + samp->shared_epilogue = epilogue; + samp->coefficients_size = (size_t)( prologue + period + epilogue ) * STBIR__COLUMN_ENTRY_FLOATS( samp->coefficient_width ) * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; + samp->contributors_size = (size_t)( prologue + period + epilogue + 1 ) * sizeof(stbir__contributors); // one per coefficient set (build time only) +} + +// the shared layout keeps contributors for just the outputs that have their own coefficient sets, in set order: +// [ P prologue outputs ][ block outputs ][ E epilogue outputs ]. +static void stbir__shared_output_range( stbir__sampler const * samp, int x, int * n0, int * n1 ) +{ + int prologue = samp->shared_prologue; + int period = samp->shared_period; + int epilogue = samp->shared_epilogue; + int num_contributors = samp->num_contributors; + stbir__contributors const * contributor; + int shift = 0; + + if ( x >= num_contributors - epilogue ) + contributor = samp->contributors + prologue + period + ( x - ( num_contributors - epilogue ) ); + else if ( x >= prologue ) + { + contributor = samp->contributors + prologue + ( ( x - prologue ) % period ); + shift = ( ( x - prologue ) / period ) * ( period / (int) samp->scale_info.scale_numerator ) * (int) samp->scale_info.scale_denominator; + } + else + contributor = samp->contributors + x; + + *n0 = contributor->n0 + shift; + *n1 = contributor->n1 + shift; +} + +// n0/n1 of output x for either layout +static void stbir__output_range( stbir__sampler const * samp, int x, int * n0, int * n1 ) +{ + if ( samp->shared_period ) + stbir__shared_output_range( samp, x, n0, n1 ); + else + { + *n0 = samp->contributors[ x ].n0; + *n1 = samp->contributors[ x ].n1; + } +} + +// The shared layout's version of stbir__cleanup_gathered_coefficients. +static void stbir__cleanup_shared_coefficients( stbir__sampler * samp ) +{ + stbir_edge edge = samp->edge; + stbir__scale_info const * scale_info = &samp->scale_info; + int input_size = scale_info->input_full_size; + int num_contributors = samp->num_contributors; + int coefficient_width = samp->coefficient_width; + int numerator = (int) scale_info->scale_numerator; + int denominator = (int) scale_info->scale_denominator; + int prologue = samp->shared_prologue; + int period = samp->shared_period; + int epilogue = samp->shared_epilogue; + stbir__contributors * contributors = samp->contributors; + float * coefficients = samp->coefficients; + float * phase_coefficients = coefficients + (size_t)prologue * coefficient_width; + stbir__filter_extent_info * extents = &samp->extent_info; + stbir_real too_small_sum = stbir__too_small_coefficient_sum_for( &samp->scale_info ); + int n, i; + + // Make the sum of each phase's coefficients 1 (as the plain path does for its first numerator outputs). + for (n = 0; n < numerator; n++) + { + stbir__contributors * contributor = contributors + n; + float * coefficient_group = phase_coefficients + (size_t)n * coefficient_width; + stbir_real total_filter = 0; + int last = contributor->n1 - contributor->n0; + + for (i = 0; i <= last; i++) + { + total_filter += (stbir_real) coefficient_group[i]; + STBIR_ASSERT( ( coefficient_group[i] >= -2.0f ) && ( coefficient_group[i] <= 2.0f ) ); // check for wonky weights + } + + if ( ( total_filter < too_small_sum ) && ( total_filter > -too_small_sum ) ) + { + // All coeffs are extremely small, just zero it. + contributor->n1 = contributor->n0; + coefficient_group[0] = 0.0f; + } + else if ( ( total_filter < ( (stbir_real)1.0 - stbir__renorm_tolerance ) ) || ( total_filter > ( (stbir_real)1.0 + stbir__renorm_tolerance ) ) ) + { + stbir_real filter_scale = (stbir_real)1.0 / total_filter; + for (i = 0; i <= last; i++) + coefficient_group[i] = (float) ( coefficient_group[i] * filter_scale ); } } - return widest; - #undef STBIR_MOVE_1 - #undef STBIR_MOVE_2 - #undef STBIR_MOVE_4 + // The contributors for the prologue and period outputs (0..prologue+period-1) are the phase's, moved along by the + // denominator once per period. The epilogue outputs (the last epilogue of the image) follow them. + for (n = numerator; n < prologue + period; n++) + { + contributors[n].n0 = contributors[n - numerator].n0 + denominator; + contributors[n].n1 = contributors[n - numerator].n1 + denominator; + } + for (n = 0; n < epilogue; n++) + { + int out_pixel = num_contributors - epilogue + n; + contributors[prologue + period + n].n0 = contributors[out_pixel % numerator].n0 + ( out_pixel / numerator ) * denominator; + contributors[prologue + period + n].n1 = contributors[out_pixel % numerator].n1 + ( out_pixel / numerator ) * denominator; + } + + // The period is the phases repeated; the prologue and epilogue groups start out as copies of their phase. + for (n = numerator; n < period; n++) + STBIR_MEMCPY( phase_coefficients + (size_t)n * coefficient_width, phase_coefficients + (size_t)( n % numerator ) * coefficient_width, coefficient_width * sizeof(float) ); + for (n = 0; n < prologue; n++) + STBIR_MEMCPY( coefficients + (size_t)n * coefficient_width, phase_coefficients + (size_t)( n % numerator ) * coefficient_width, coefficient_width * sizeof(float) ); + for (n = 0; n < epilogue; n++) + STBIR_MEMCPY( coefficients + (size_t)( prologue + period + n ) * coefficient_width, phase_coefficients + (size_t)( ( num_contributors - epilogue + n ) % numerator ) * coefficient_width, coefficient_width * sizeof(float) ); + + // Fold and trim every group. The period's outputs are never at an edge (that is what the prologue and epilogue are + // for), so for them this is just the trim, which is the same for every copy of a phase - the period stays periodic. + extents->lowest = 0x7fffffff; + extents->highest = -0x7fffffff; + extents->widest = -1; + for (n = 0; n < prologue + period + epilogue; n++) + { + STBIR_ASSERT( ( n < prologue ) || ( n >= prologue + period ) || ( edge == STBIR_EDGE_WRAP ) || ( ( contributors[n].n0 >= 0 ) && ( contributors[n].n1 < input_size ) ) ); + stbir__fold_and_trim_contributor( edge, contributors + n, coefficients + (size_t)n * coefficient_width, coefficient_width, input_size, extents ); + } + + // lowest and highest are over every output: the extremes are in the prologue and first period, and the last period + // and epilogue. + { + int n0, n1; + stbir__shared_output_range( samp, num_contributors - epilogue - 1, &n0, &n1 ); + if ( n1 > extents->highest ) + extents->highest = n1; + } } -static void stbir__calculate_filters( stbir__sampler * samp, stbir__sampler * other_axis_for_pivot, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) +// the shared layout's version of stbir__pack_coefficients +static int stbir__pack_shared_coefficients( stbir__sampler * samp, int row0, int row1 ) { + int num_contributors = samp->num_contributors; + int widest = samp->extent_info.widest; + int entry_floats = STBIR__COLUMN_ENTRY_FLOATS( widest ); + int prologue = samp->shared_prologue; + int period = samp->shared_period; + int epilogue = samp->shared_epilogue; + int n0, n1; + + stbir__pack_column_entries( prologue + period + epilogue, samp->coefficients, samp->coefficient_width, widest ); + stbir__clip_contributor_reads( epilogue, samp->contributors + prologue + period, samp->coefficients + (size_t)( prologue + period ) * entry_floats + 2, entry_floats, widest, row0, row1 ); + + // the clipping never needed to reach past the epilogue (the last periodic output is where it would have gone next) + stbir__shared_output_range( samp, num_contributors - epilogue - 1, &n0, &n1 ); + STBIR_ASSERT( ( n0 + widest * 2 ) < row1 + 1 ); + return widest; +} + +// the packed column entry at an index (the entries are entry_floats apart) +static stbir__column_entry * stbir__column_entry_at( stbir__sampler const * samp, int index ) +{ + return (stbir__column_entry *)( samp->coefficients + (size_t)index * samp->entry_floats ); +} + +// fill in the column entries' advance and width from the build time contributors (after the entries have been +// packed, and after the contributors have been read for the last time - the column slice table). returns 0 when +// an advance doesn't fit the entry's int - a horizontal downsample of around 2^29:1 at four channels +static int stbir__fill_column_entries( stbir__sampler * samp, int channels ) +{ + stbir__contributors const * contributors = samp->contributors; + int num_entries = samp->shared_period ? ( samp->shared_prologue + samp->shared_period + samp->shared_epilogue ) : samp->num_contributors; + int i; + + if ( samp->shared_period ) + { + int prologue = samp->shared_prologue; + int period = samp->shared_period; + samp->shared_block_n0 = contributors[prologue].n0; + samp->shared_block_shift = ( period / (int) samp->scale_info.scale_numerator ) * (int) samp->scale_info.scale_denominator; + samp->shared_epilogue_n0 = contributors[prologue + period].n0; + } + + // each entry says how far the decode pointer moves to the NEXT column. the block's last entry says how far to the + // block's first column one pass on (the wrap) + for (i = 0; i < num_entries; i++) + { + stbir__column_entry * entry = stbir__column_entry_at( samp, i ); + int advance; + + if ( ( samp->shared_period ) && ( i == samp->shared_prologue + samp->shared_period - 1 ) ) + advance = ( samp->shared_block_n0 + samp->shared_block_shift ) - contributors[i].n0; + else + advance = ( i + 1 < num_entries ) ? ( contributors[i + 1].n0 - contributors[i].n0 ) : 0; + + if ( ( (stbir_int64)advance * channels <= -0x7fffffff ) || ( (stbir_int64)advance * channels >= 0x7fffffff ) ) + return 0; // error + + entry->advance = advance * channels; + entry->width = contributors[i].n1 - contributors[i].n0 + 1; + } + return 1; +} + +// which coefficient set (column entry) output x uses, for either layout +static int stbir__output_entry_index( stbir__sampler const * samp, int x ) +{ + if ( samp->shared_period ) + { + int prologue = samp->shared_prologue; + int period = samp->shared_period; + int epilogue = samp->shared_epilogue; + int num_contributors = samp->num_contributors; + + if ( x >= num_contributors - epilogue ) + return prologue + period + ( x - ( num_contributors - epilogue ) ); + if ( x >= prologue ) + return prologue + ( ( x - prologue ) % period ); + } + return x; +} + +static float const * stbir__output_coefficients( stbir__sampler const * samp, int x ) +{ + return samp->coefficients + (size_t)stbir__output_entry_index( samp, x ) * samp->coefficient_width; +} + +// output x's packed column entry +static stbir__column_entry const * stbir__output_column_entry( stbir__sampler const * samp, int x ) +{ + return stbir__column_entry_at( samp, stbir__output_entry_index( samp, x ) ); +} + +// Vertical scatter functions - used when we are downsampling around 32:1 or more (horizontal is always gather) +// advice-wise, if you are downsampling this amount, then you'd better off doing the downsample in a couple steps +static void stbir__calculate_coefficients_scatter( stbir__kernel_callback_new * kernel_new, stbir__kernel_callback * kernel_old, stbir__scale_info const * scale_info, int in_pixel, int out_first_pixel, int out_last_pixel, stbir__contributors* contributor, float* coefficient_group, void * user_data ) +{ + int i; + + contributor->n0 = out_first_pixel; + contributor->n1 = out_last_pixel; + + for (i = 0; i <= out_last_pixel - out_first_pixel; i++) + { + stbir_real weight = stbir__kernel_weight( kernel_new, kernel_old, stbir__tap_offset( scale_info, i + out_first_pixel, in_pixel, 0 ), scale_info->scale, user_data ); + float coeff = 0.0f; + + if ( ( weight >= stbir__too_small_coefficient ) || ( weight <= -stbir__too_small_coefficient ) ) + coeff = (float) ( weight * scale_info->scale ); + + coefficient_group[i] = coeff; + } +} + +// Normalize each output's coefficients: summed and scaled in stbir_real, each then rounded to float +static void stbir__normalize_scatter_coefficients( stbir__contributors* contributors, float* coefficients, int coefficient_width, int num_contributors, int output_size, stbir_real too_small_sum ) +{ + int i, j, first = 0; + + for (i = 0; i < output_size; i++) + { + stbir_real total = 0; + stbir_real filter_scale; + + while ( ( first < num_contributors ) && ( contributors[first].n1 < i ) ) + first++; + + for (j = first; j < num_contributors; j++) + { + if (i >= contributors[j].n0 && i <= contributors[j].n1) + total += (stbir_real) coefficients[ (size_t)j * coefficient_width + i - contributors[j].n0 ]; + else if (i < contributors[j].n0) + break; + } + + if ( ( total < too_small_sum ) && ( total > -too_small_sum ) ) + filter_scale = 0; // all coeffs are extremely small, just zero them + else if ( ( total < ( (stbir_real)1.0 - stbir__renorm_tolerance ) ) || ( total > ( (stbir_real)1.0 + stbir__renorm_tolerance ) ) ) + filter_scale = (stbir_real)1.0 / total; + else + continue; + + for (j = first; j < num_contributors; j++) + { + if (i >= contributors[j].n0 && i <= contributors[j].n1) + coefficients[ (size_t)j * coefficient_width + i - contributors[j].n0 ] = (float) ( coefficients[ (size_t)j * coefficient_width + i - contributors[j].n0 ] * filter_scale ); + else if (i < contributors[j].n0) + break; + } + } +} + +// Polyphase: only the first numerator outputs were calculated. From first_copy on, each contributor is the one a +// denominator before it with its outputs moved along by the numerator (calculated ones keep their outputs, the copies follow). +static void stbir__replicate_scatter_coefficients( stbir__contributors* contributors, float* coefficients, int coefficient_width, int num_contributors, int first_copy, int calculated_end, int numerator, int denominator, int output_size ) +{ + int j, i; + + for (j = first_copy; j < num_contributors; j++) + { + stbir__contributors* contributor = contributors + j; + stbir__contributors* source = contributors + j - denominator; + float* coefficient_group = coefficients + (size_t)j * coefficient_width; + float* source_group = coefficients + (size_t)( j - denominator ) * coefficient_width; + + if ( j >= calculated_end ) + { + contributor->n0 = 0; + contributor->n1 = -1; + + // the whole source fits: copy it + if ( source->n1 + numerator < output_size ) + { + contributor->n0 = source->n0 + numerator; + contributor->n1 = source->n1 + numerator; + STBIR_MEMCPY( coefficient_group, source_group, (size_t)( source->n1 - source->n0 + 1 ) * sizeof(float) ); + continue; + } + } + + // append the source's outputs that are in range + for (i = 0; i <= source->n1 - source->n0; i++) + { + int out = source->n0 + i + numerator; + if ( out >= output_size ) + break; + if ( contributor->n1 < contributor->n0 ) + { + contributor->n0 = contributor->n1 = out; + coefficient_group[0] = source_group[i]; + } + else + stbir__insert_coeff( contributor, coefficient_group, out, source_group[i], coefficient_width ); + } + } +} + +// The out of range input scanlines: clamp and reflect add their weights to the scanlines they map to (right edge +// ascending, then left edge descending - the gather's order), zero drops them, wrap keeps them. +static void stbir__fold_scatter_edges( stbir__sampler * samp, int coefficient_width ) +{ + stbir_edge edge = samp->edge; + int input_size = samp->scale_info.input_full_size; + int filter_pixel_margin = samp->filter_pixel_margin; + stbir__contributors* contributors = samp->contributors; + float* coefficients = samp->coefficients; + int n, i; + + if ( ( edge == STBIR_EDGE_CLAMP ) || ( edge == STBIR_EDGE_REFLECT ) ) + { + int pass; + for (pass = 0; pass < 2; pass++) + { + int from = pass ? -1 : input_size; + int to = pass ? -filter_pixel_margin - 1 : input_size + filter_pixel_margin; + int step = pass ? -1 : 1; + for (n = from; n != to; n += step) + { + stbir__contributors* contributor = contributors + n + filter_pixel_margin; + float* coefficient_group = coefficients + (size_t)( n + filter_pixel_margin ) * coefficient_width; + int folded = stbir__edge_wrap_slow[edge]( n, input_size ) + filter_pixel_margin; + + for (i = 0; i <= contributor->n1 - contributor->n0; i++) + stbir__insert_coeff( contributors + folded, coefficients + (size_t)folded * coefficient_width, contributor->n0 + i, coefficient_group[i], coefficient_width ); + + contributor->n0 = 0; + contributor->n1 = -1; + } + } + } + else if ( edge == STBIR_EDGE_ZERO ) + { + for (n = 0; n < filter_pixel_margin; n++) + { + contributors[n].n0 = 0; + contributors[n].n1 = -1; + contributors[input_size + 2 * filter_pixel_margin - 1 - n].n0 = 0; + contributors[input_size + 2 * filter_pixel_margin - 1 - n].n1 = -1; + } + } +} + +// Optimize step - skip zero coefficients at either end, and pack the groups from the build width down to the final width +// (an output past the width is dropped). Also notes the first and last input scanlines with any weight. +static void stbir__pack_scatter_coefficients( stbir__sampler * samp, int build_width ) +{ + int coefficient_width = samp->coefficient_width; + int filter_pixel_margin = samp->filter_pixel_margin; + int num_contributors = samp->num_contributors; + stbir__contributors* contributors = samp->contributors; + float* coefficients = samp->coefficients; + int j, i; + + samp->extent_info.lowest = 0x7fffffff; + samp->extent_info.highest = -0x7fffffff; + samp->extent_info.widest = coefficient_width; + + for (j = 0; j < num_contributors; j++) + { + stbir__contributors* contributor = contributors + j; + float const* source_group = coefficients + (size_t)j * build_width; + float* coefficient_group = coefficients + (size_t)j * coefficient_width; // at or before the source, so packing forward in place is safe + int range = contributor->n1 - contributor->n0 + 1; + int skip = 0; + + while ( ( skip < range ) && ( source_group[skip] == 0.0f ) ) + skip++; + range -= skip; + if ( range > coefficient_width ) + range = coefficient_width; + while ( ( range > 0 ) && ( source_group[skip + range - 1] == 0.0f ) ) + range--; + + for (i = 0; i < range; i++) + coefficient_group[i] = source_group[i + skip]; + + if ( range <= 0 ) + { + contributor->n0 = 0; + contributor->n1 = -1; + continue; + } + contributor->n0 += skip; + contributor->n1 = contributor->n0 + range - 1; + + if ( j - filter_pixel_margin < samp->extent_info.lowest ) + samp->extent_info.lowest = j - filter_pixel_margin; + samp->extent_info.highest = j - filter_pixel_margin; + } +} + +static void stbir__calculate_scatter_filters( stbir__sampler * samp, stbir_real in_pixels_radius, stbir_real support, void * user_data ) +{ + stbir__scale_info const * scale_info = &samp->scale_info; + int output_size = scale_info->output_sub_size; + int filter_pixel_margin = samp->filter_pixel_margin; + int num_contributors = samp->num_contributors; + int build_width = samp->coefficient_width + 1; + int polyphase = ( scale_info->scale_is_rational ) && ( (int)scale_info->scale_numerator < output_size ); + int numerator = polyphase ? (int)scale_info->scale_numerator : output_size; + int calculated_end = num_contributors; // the first contributor with nothing calculated + int first_weighted = -1; // the first contributor with any weight int n; - float scale = samp->scale_info.scale; - stbir__kernel_callback * kernel = samp->filter_kernel; - stbir__support_callback * support = samp->filter_support; - float inv_scale = samp->scale_info.inv_scale; + + // Looping through in pixels + for (n = 0; n < num_contributors; n++) + { + int out_first_pixel, out_last_pixel; + + stbir__input_out_pixel_range( scale_info, n - filter_pixel_margin, in_pixels_radius, support, &out_first_pixel, &out_last_pixel ); + + // when polyphase, you only have to do coeffs up to the numerator count + if ( out_first_pixel >= numerator ) + { + calculated_end = n; + break; + } + if ( out_last_pixel >= numerator ) + out_last_pixel = numerator - 1; + + stbir__calculate_coefficients_scatter( samp->filter_kernel_new, samp->filter_kernel_old, scale_info, n - filter_pixel_margin, out_first_pixel, out_last_pixel, samp->contributors + n, samp->coefficients + (size_t)n * build_width, user_data ); + + if ( ( first_weighted < 0 ) && ( out_last_pixel >= out_first_pixel ) ) + first_weighted = n; + } + for (n = calculated_end; n < num_contributors; n++) + { + samp->contributors[n].n0 = 0; + samp->contributors[n].n1 = -1; + } + + stbir__normalize_scatter_coefficients( samp->contributors, samp->coefficients, build_width, calculated_end, numerator, stbir__too_small_coefficient_sum_for( scale_info ) ); + + // the period is the rational's, which can be a little off the true one, so the copies start where it says + if ( ( polyphase ) && ( first_weighted >= 0 ) ) + stbir__replicate_scatter_coefficients( samp->contributors, samp->coefficients, build_width, num_contributors, first_weighted + (int)scale_info->scale_denominator, calculated_end, numerator, (int)scale_info->scale_denominator, output_size ); + + stbir__fold_scatter_edges( samp, build_width ); + stbir__pack_scatter_coefficients( samp, build_width ); +} + +static void stbir__calculate_filters( stbir__sampler * samp, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) +{ + stbir_real scale = samp->scale_info.scale; + stbir__kernel_callback_new * kernel_new = samp->filter_kernel_new; + stbir__kernel_callback * kernel_old = samp->filter_kernel_old; + stbir_real in_pixels_radius = stbir__filter_radius_in_input_pixels( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, &samp->scale_info, user_data ); + stbir_real inv_scale = samp->scale_info.inv_scale; int input_full_size = samp->scale_info.input_full_size; int gather_num_contributors = samp->num_contributors; stbir__contributors* gather_contributors = samp->contributors; float * gather_coeffs = samp->coefficients; int gather_coefficient_width = samp->coefficient_width; + // with shared polyphase sets the generated phases go after the prologue sets + if ( samp->shared_period ) + gather_coeffs += (size_t)samp->shared_prologue * gather_coefficient_width; + switch ( samp->is_gather ) { case 1: // gather upsample { - float out_pixels_radius = support(inv_scale,user_data) * scale; + stbir_real filter_support = in_pixels_radius; // the filter's own pixels are input pixels + stbir_real out_pixels_radius = filter_support * scale; - stbir__calculate_coefficients_for_gather_upsample( out_pixels_radius, kernel, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width, samp->edge, user_data ); + stbir__calculate_coefficients_for_gather_upsample( out_pixels_radius, filter_support, kernel_new, kernel_old, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width, samp->edge, user_data ); STBIR_PROFILE_BUILD_START( cleanup ); - stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); + if ( samp->shared_period ) + stbir__cleanup_shared_coefficients( samp ); + else + stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); STBIR_PROFILE_BUILD_END( cleanup ); } break; @@ -3958,118 +4689,33 @@ static void stbir__calculate_filters( stbir__sampler * samp, stbir__sampler * ot case 0: // scatter downsample (only on vertical) case 2: // gather downsample { - float in_pixels_radius = support(scale,user_data) * inv_scale; + stbir_real filter_support = in_pixels_radius / inv_scale; // in output pixels int filter_pixel_margin = samp->filter_pixel_margin; int input_end = input_full_size + filter_pixel_margin; - // if this is a scatter, we do a downsample gather to get the coeffs, and then pivot after if ( !samp->is_gather ) { - // check if we are using the same gather downsample on the horizontal as this vertical, - // if so, then we don't have to generate them, we can just pivot from the horizontal. - if ( other_axis_for_pivot ) - { - gather_contributors = other_axis_for_pivot->contributors; - gather_coeffs = other_axis_for_pivot->coefficients; - gather_coefficient_width = other_axis_for_pivot->coefficient_width; - gather_num_contributors = other_axis_for_pivot->num_contributors; - samp->extent_info.lowest = other_axis_for_pivot->extent_info.lowest; - samp->extent_info.highest = other_axis_for_pivot->extent_info.highest; - samp->extent_info.widest = other_axis_for_pivot->extent_info.widest; - goto jump_right_to_pivot; - } - - gather_contributors = samp->gather_prescatter_contributors; - gather_coeffs = samp->gather_prescatter_coefficients; - gather_coefficient_width = samp->gather_prescatter_coefficient_width; - gather_num_contributors = samp->gather_prescatter_num_contributors; + STBIR_PROFILE_BUILD_START( scatter ); + stbir__calculate_scatter_filters( samp, in_pixels_radius, filter_support, user_data ); + STBIR_PROFILE_BUILD_END( scatter ); + break; } - stbir__calculate_coefficients_for_gather_downsample( -filter_pixel_margin, input_end, in_pixels_radius, kernel, &samp->scale_info, gather_coefficient_width, gather_num_contributors, gather_contributors, gather_coeffs, user_data ); + stbir__calculate_coefficients_for_gather_downsample( -filter_pixel_margin, input_end, in_pixels_radius, filter_support, kernel_new, kernel_old, &samp->scale_info, gather_coefficient_width, gather_num_contributors, gather_contributors, gather_coeffs, user_data ); STBIR_PROFILE_BUILD_START( cleanup ); - stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); + if ( samp->shared_period ) + stbir__cleanup_shared_coefficients( samp ); + else + stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); STBIR_PROFILE_BUILD_END( cleanup ); - - if ( !samp->is_gather ) - { - // if this is a scatter (vertical only), then we need to pivot the coeffs - stbir__contributors * scatter_contributors; - int highest_set; - - jump_right_to_pivot: - - STBIR_PROFILE_BUILD_START( pivot ); - - highest_set = (-filter_pixel_margin) - 1; - for (n = 0; n < gather_num_contributors; n++) - { - int k; - int gn0 = gather_contributors->n0, gn1 = gather_contributors->n1; - int scatter_coefficient_width = samp->coefficient_width; - float * scatter_coeffs = samp->coefficients + ( gn0 + filter_pixel_margin ) * scatter_coefficient_width; - float * g_coeffs = gather_coeffs; - scatter_contributors = samp->contributors + ( gn0 + filter_pixel_margin ); - - for (k = gn0 ; k <= gn1 ; k++ ) - { - float gc = *g_coeffs++; - - // skip zero and denormals - must skip zeros to avoid adding coeffs beyond scatter_coefficient_width - // (which happens when pivoting from horizontal, which might have dummy zeros) - if ( ( ( gc >= stbir__small_float ) || ( gc <= -stbir__small_float ) ) ) - { - if ( ( k > highest_set ) || ( scatter_contributors->n0 > scatter_contributors->n1 ) ) - { - { - // if we are skipping over several contributors, we need to clear the skipped ones - stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1); - while ( clear_contributors < scatter_contributors ) - { - clear_contributors->n0 = 0; - clear_contributors->n1 = -1; - ++clear_contributors; - } - } - scatter_contributors->n0 = n; - scatter_contributors->n1 = n; - scatter_coeffs[0] = gc; - highest_set = k; - } - else - { - stbir__insert_coeff( scatter_contributors, scatter_coeffs, n, gc, scatter_coefficient_width ); - } - STBIR_ASSERT( ( scatter_contributors->n1 - scatter_contributors->n0 + 1 ) <= scatter_coefficient_width ); - } - ++scatter_contributors; - scatter_coeffs += scatter_coefficient_width; - } - - ++gather_contributors; - gather_coeffs += gather_coefficient_width; - } - - // now clear any unset contribs - { - stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1); - stbir__contributors * end_contributors = samp->contributors + samp->num_contributors; - while ( clear_contributors < end_contributors ) - { - clear_contributors->n0 = 0; - clear_contributors->n1 = -1; - ++clear_contributors; - } - } - - STBIR_PROFILE_BUILD_END( pivot ); - } } break; } } + //======================================================================================================== // scanline decoders and encoders @@ -4135,7 +4781,7 @@ static void stbir__calculate_filters( stbir__sampler * samp, stbir__sampler * ot // fancy alpha means we expand to keep both premultipied and non-premultiplied color channels -static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_channels ) +static void stbir__fancy_alpha_weight_4ch( float * out_buffer, ptrdiff_t width_times_channels ) { float STBIR_STREAMOUT_PTR(*) out = out_buffer; float const * end_decode = out_buffer + ( width_times_channels / 4 ) * 7; // decode buffer aligned to end of out_buffer @@ -4231,7 +4877,7 @@ static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_c #endif } -static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_channels ) +static void stbir__fancy_alpha_weight_2ch( float * out_buffer, ptrdiff_t width_times_channels ) { float STBIR_STREAMOUT_PTR(*) out = out_buffer; float const * end_decode = out_buffer + ( width_times_channels / 2 ) * 3; @@ -4301,7 +4947,7 @@ static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_c } } -static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_times_channels ) +static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, ptrdiff_t width_times_channels ) { float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer; @@ -4315,7 +4961,7 @@ static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_ti #ifdef STBIR_SIMD stbir__simdf i,ia; STBIR_SIMD_NO_UNROLL(encode); - if ( alpha < stbir__small_float ) + if ( alpha < stbir__alpha_too_small_to_divide ) { stbir__simdf_load( i, input ); stbir__simdf_store( encode, i ); @@ -4329,7 +4975,7 @@ static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_ti encode[3] = alpha; } #else - if ( alpha < stbir__small_float ) + if ( alpha < stbir__alpha_too_small_to_divide ) { encode[0] = input[0]; encode[1] = input[1]; @@ -4351,7 +4997,7 @@ static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_ti } // format: [X A Xpm][X A Xpm] etc -static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, int width_times_channels ) +static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, ptrdiff_t width_times_channels ) { float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer; @@ -4360,7 +5006,7 @@ static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, int width_ti do { float alpha = input[1]; encode[0] = input[0]; - if ( alpha >= stbir__small_float ) + if ( alpha >= stbir__alpha_too_small_to_divide ) encode[0] = input[2] / alpha; encode[1] = alpha; @@ -4369,7 +5015,7 @@ static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, int width_ti } while ( encode < end_output ); } -static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_times_channels ) +static void stbir__simple_alpha_weight_4ch( float * decode_buffer, ptrdiff_t width_times_channels ) { float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; float const * end_decode = decode_buffer + width_times_channels; @@ -4425,7 +5071,7 @@ static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_tim #endif } -static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_times_channels ) +static void stbir__simple_alpha_weight_2ch( float * decode_buffer, ptrdiff_t width_times_channels ) { float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; float const * end_decode = decode_buffer + width_times_channels; @@ -4460,7 +5106,7 @@ static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_tim } } -static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_times_channels ) +static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, ptrdiff_t width_times_channels ) { float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; float const * end_output = encode_buffer + width_times_channels; @@ -4472,7 +5118,7 @@ static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_t #ifdef STBIR_SIMD stbir__simdf i,ia; STBIR_SIMD_NO_UNROLL(encode); - if ( alpha >= stbir__small_float ) + if ( alpha >= stbir__alpha_too_small_to_divide ) { stbir__simdf_load1frep4( ia, 1.0f / alpha ); stbir__simdf_load( i, encode ); @@ -4481,7 +5127,7 @@ static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_t encode[3] = alpha; } #else - if ( alpha >= stbir__small_float ) + if ( alpha >= stbir__alpha_too_small_to_divide ) { float ialpha = 1.0f / alpha; encode[0] *= ialpha; @@ -4493,14 +5139,14 @@ static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_t } while ( encode < end_output ); } -static void stbir__simple_alpha_unweight_2ch( float * encode_buffer, int width_times_channels ) +static void stbir__simple_alpha_unweight_2ch( float * encode_buffer, ptrdiff_t width_times_channels ) { float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; float const * end_output = encode_buffer + width_times_channels; do { float alpha = encode[1]; - if ( alpha >= stbir__small_float ) + if ( alpha >= stbir__alpha_too_small_to_divide ) encode[0] /= alpha; encode += 2; } while ( encode < end_output ); @@ -4508,7 +5154,7 @@ static void stbir__simple_alpha_unweight_2ch( float * encode_buffer, int width_t // only used in RGB->BGR or BGR->RGB -static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_channels ) +static void stbir__simple_flip_3ch( float * decode_buffer, ptrdiff_t width_times_channels ) { float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; float const * end_decode = decode_buffer + width_times_channels; @@ -4586,9 +5232,18 @@ static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_chann // 16 instructions float t0,t1,t2,t3; STBIR_NO_UNROLL(decode); - t0 = decode[0]; t1 = decode[3]; t2 = decode[6]; t3 = decode[9]; - decode[0] = decode[2]; decode[3] = decode[5]; decode[6] = decode[8]; decode[9] = decode[11]; - decode[2] = t0; decode[5] = t1; decode[8] = t2; decode[11] = t3; + t0 = decode[0]; + t1 = decode[3]; + t2 = decode[6]; + t3 = decode[9]; + decode[0] = decode[2]; + decode[3] = decode[5]; + decode[6] = decode[8]; + decode[9] = decode[11]; + decode[2] = t0; + decode[5] = t1; + decode[8] = t2; + decode[11] = t3; decode += 12; } end_decode += 12; @@ -4606,47 +5261,82 @@ static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_chann } - -static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float * output_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +// decodes the input pixels that a column slice needs for input scanline n into output_buffer +// most of the complexity in this function is for wrap mode, we need to grab from multiple +// points in the input buffer +static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float * output_buffer, stbir__column_slice const * slice STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) { int channels = stbir_info->channels; int effective_channels = stbir_info->effective_channels; int input_sample_in_bytes = stbir__type_size[stbir_info->input_type] * channels; stbir_edge edge_horizontal = stbir_info->horizontal.edge; stbir_edge edge_vertical = stbir_info->vertical.edge; + int input_full_size = stbir_info->horizontal.scale_info.input_full_size; int row = stbir__edge_wrap(edge_vertical, n, stbir_info->vertical.scale_info.input_full_size); - const void* input_plane_data = ( (char *) stbir_info->input_data ) + (size_t)row * (size_t) stbir_info->input_stride_bytes; - stbir__span const * spans = stbir_info->scanline_extents.spans; - float * full_decode_buffer = output_buffer - stbir_info->scanline_extents.conservative.n0 * effective_channels; - float * last_decoded = 0; + const void* input_plane_data = ( (char *) stbir_info->input_data ) + (ptrdiff_t)( row * stbir_info->input_stride_bytes ); + float * full_decode_buffer = output_buffer - (ptrdiff_t)slice->in_n0 * effective_channels; // decode buffer is indexed in input pixel space, so back it up to input pixel zero (the products are 64-bit: a tile of a huge input can start past 2^31 / channels) + float * last_decoded = output_buffer; + int run_n0[3], run_n1[3], run_input_x[3]; + int num_runs = 0, r; + int n0 = slice->in_n0, n1 = slice->in_n1; // if we are on edge_zero, and we get in here with an out of bounds n, then the calculate filters has failed STBIR_ASSERT( !(edge_vertical == STBIR_EDGE_ZERO && (n < 0 || n >= stbir_info->vertical.scale_info.input_full_size)) ); - do + if ( edge_horizontal == STBIR_EDGE_WRAP ) { - float * decode_buffer; + // the margins are never wider than the scanline (stbir__set_sampler clamps them), so each side wraps exactly once + STBIR_ASSERT( n0 >= -input_full_size ); + if ( n1 > ( ( input_full_size - 1 ) + input_full_size ) ) + n1 = ( input_full_size - 1 ) + input_full_size; // the conservative range can poke one past what the filter can reach, that pixel just gets zeroed below ( 2 x 2^30 - 1, written so it can't overflow ) + + if ( n0 < 0 ) + { + run_n0[num_runs] = n0; + run_n1[num_runs] = ( n1 < -1 ) ? n1 : -1; + run_input_x[num_runs] = n0 + input_full_size; + ++num_runs; + } + if ( ( n1 >= 0 ) && ( n0 < input_full_size ) ) + { + run_n0[num_runs] = ( n0 > 0 ) ? n0 : 0; + run_n1[num_runs] = ( n1 < ( input_full_size - 1 ) ) ? n1 : ( input_full_size - 1 ); + run_input_x[num_runs] = run_n0[num_runs]; + ++num_runs; + } + if ( n1 >= input_full_size ) + { + run_n0[num_runs] = ( n0 > input_full_size ) ? n0 : input_full_size; + run_n1[num_runs] = n1; + run_input_x[num_runs] = run_n0[num_runs] - input_full_size; + ++num_runs; + } + } + else + { + STBIR_ASSERT( ( n0 >= 0 ) && ( n1 < input_full_size ) ); + run_n0[0] = n0; + run_n1[0] = n1; + run_input_x[0] = n0; + num_runs = 1; + } + + for( r = 0 ; r < num_runs ; r++ ) + { + int width = run_n1[r] + 1 - run_n0[r]; + ptrdiff_t width_times_channels = (ptrdiff_t)width * channels; + float * decode_buffer = full_decode_buffer + (ptrdiff_t)run_n0[r] * effective_channels; + float * end_decode = full_decode_buffer + (ptrdiff_t)( run_n1[r] + 1 ) * effective_channels; void const * input_data; - float * end_decode; - int width_times_channels; - int width; - - if ( spans->n1 < spans->n0 ) - break; - - width = spans->n1 + 1 - spans->n0; - decode_buffer = full_decode_buffer + spans->n0 * effective_channels; - end_decode = full_decode_buffer + ( spans->n1 + 1 ) * effective_channels; - width_times_channels = width * channels; // read directly out of input plane by default - input_data = ( (char*)input_plane_data ) + spans->pixel_offset_for_input * input_sample_in_bytes; + input_data = ( (char*)input_plane_data ) + (ptrdiff_t)run_input_x[r] * input_sample_in_bytes; // if we have an input callback, call it to get the input data if ( stbir_info->in_pixels_cb ) { // call the callback with a temp buffer (that they can choose to use or not). the temp is just right aligned memory in the decode_buffer itself - input_data = stbir_info->in_pixels_cb( ( (char*) end_decode ) - ( width * input_sample_in_bytes ) + ( ( stbir_info->input_type != STBIR_TYPE_FLOAT ) ? ( sizeof(float)*STBIR_INPUT_CALLBACK_PADDING ) : 0 ), input_plane_data, width, spans->pixel_offset_for_input, row, stbir_info->user_data ); + input_data = stbir_info->in_pixels_cb( ( (char*) end_decode ) - ( (ptrdiff_t)width * input_sample_in_bytes ) + stbir_info->input_callback_offset_bytes, input_plane_data, width, run_input_x[r], row, stbir_info->user_data ); } STBIR_PROFILE_START( decode ); @@ -4660,47 +5350,23 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir_info->alpha_weight( decode_buffer, width_times_channels ); STBIR_PROFILE_END( alpha ); } - - ++spans; - } while ( spans <= ( &stbir_info->scanline_extents.spans[1] ) ); - - // handle the edge_wrap filter (all other types are handled back out at the calculate_filter stage) - // basically the idea here is that if we have the whole scanline in memory, we don't redecode the - // wrapped edge pixels, and instead just memcpy them from the scanline into the edge positions - if ( ( edge_horizontal == STBIR_EDGE_WRAP ) && ( stbir_info->scanline_extents.edge_sizes[0] | stbir_info->scanline_extents.edge_sizes[1] ) ) - { - // this code only runs if we're in edge_wrap, and we're doing the entire scanline - int e, start_x[2]; - int input_full_size = stbir_info->horizontal.scale_info.input_full_size; - - start_x[0] = -stbir_info->scanline_extents.edge_sizes[0]; // left edge start x - start_x[1] = input_full_size; // right edge - - for( e = 0; e < 2 ; e++ ) - { - // do each margin - int margin = stbir_info->scanline_extents.edge_sizes[e]; - if ( margin ) - { - int x = start_x[e]; - float * marg = full_decode_buffer + x * effective_channels; - float const * src = full_decode_buffer + stbir__edge_wrap(edge_horizontal, x, input_full_size) * effective_channels; - STBIR_MEMCPY( marg, src, margin * effective_channels * sizeof(float) ); - if ( e == 1 ) last_decoded = marg + margin * effective_channels; - } - } } - - // some of the horizontal gathers read one float off the edge (which is masked out), but we force a zero here to make sure no NaNs leak in - // (we can't pre-zero it, because the input callback can use that area as padding) - last_decoded[0] = 0.0f; - // we clear this extra float, because the final output pixel filter kernel might have used one less coeff than the max filter width - // when this happens, we do read that pixel from the input, so it too could be Nan, so just zero an extra one. - // this fits because each scanline is padded by three floats (STBIR_INPUT_CALLBACK_PADDING) - last_decoded[1] = 0.0f; + // the horizontal kernels sometimes read a few samples past the input end, so + // zero the extra to avoid NaN (has to happen after decoding, since the input + // buffer might overwrite them. Should just be a couple clears. + { + float * range_end = full_decode_buffer + (ptrdiff_t)( slice->in_n1 + 1 ) * effective_channels + 2; + while ( last_decoded < range_end ) + *last_decoded++ = 0.0f; + } } +// advance to the next set of coeffs and contrib info +#define stbir__next_contributor() \ + decode_buffer += column->advance; \ + column = ( (stbir__column_entry const *)( ( (char const *)(column) ) + (entry_stride) ) ); + //================= // Do 1 channel horizontal routines @@ -4734,8 +5400,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output_tiny() \ stbir__simdf_store1( output, tot ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 1; #define stbir__4_coeff_start() \ @@ -4770,14 +5435,214 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_and( c, c, mask ); \ stbir__simdf_madd_mem( tot, tot, c, decode+(ofs) ); +// four outputs per iteration for one channel - do 4 1-channel versions +#define stbir__x4_setup() \ + stbir__column_entry const * e1 = (stbir__column_entry const *)( (char const *)column + entry_stride ); \ + stbir__column_entry const * e2 = (stbir__column_entry const *)( (char const *)e1 + entry_stride ); \ + stbir__column_entry const * e3 = (stbir__column_entry const *)( (char const *)e2 + entry_stride ); \ + float const * d0 = decode_buffer, * d1 = d0 + column->advance, * d2 = d1 + e1->advance, * d3 = d2 + e2->advance; \ + float const * h0 = column->coefficients, * h1 = e1->coefficients, * h2 = e2->coefficients, * h3 = e3->coefficients; \ + stbir__simdf a0, a1, a2, a3, c; + +#define stbir__x4_start() \ + do { \ + stbir__simdf_load( c, h0 ); \ + stbir__simdf_mult_mem( a0, c, d0 ); \ + stbir__simdf_load( c, h1 ); \ + stbir__simdf_mult_mem( a1, c, d1 ); \ + stbir__simdf_load( c, h2 ); \ + stbir__simdf_mult_mem( a2, c, d2 ); \ + stbir__simdf_load( c, h3 ); \ + stbir__simdf_mult_mem( a3, c, d3 ); \ + } while(0) + +#define stbir__x4_cont( ofs ) \ + do { \ + stbir__simdf_load( c, h0 + (ofs) ); \ + stbir__simdf_madd_mem( a0, a0, c, d0 + (ofs) ); \ + stbir__simdf_load( c, h1 + (ofs) ); \ + stbir__simdf_madd_mem( a1, a1, c, d1 + (ofs) ); \ + stbir__simdf_load( c, h2 + (ofs) ); \ + stbir__simdf_madd_mem( a2, a2, c, d2 + (ofs) ); \ + stbir__simdf_load( c, h3 + (ofs) ); \ + stbir__simdf_madd_mem( a3, a3, c, d3 + (ofs) ); \ + } while(0) + +#define stbir__x4_rem1( ofs ) \ + do { \ + stbir__simdf d; \ + stbir__simdf_load1z( c, h0 + (ofs) ); \ + stbir__simdf_load1( d, d0 + (ofs) ); \ + stbir__simdf_madd( a0, a0, d, c ); \ + stbir__simdf_load1z( c, h1 + (ofs) ); \ + stbir__simdf_load1( d, d1 + (ofs) ); \ + stbir__simdf_madd( a1, a1, d, c ); \ + stbir__simdf_load1z( c, h2 + (ofs) ); \ + stbir__simdf_load1( d, d2 + (ofs) ); \ + stbir__simdf_madd( a2, a2, d, c ); \ + stbir__simdf_load1z( c, h3 + (ofs) ); \ + stbir__simdf_load1( d, d3 + (ofs) ); \ + stbir__simdf_madd( a3, a3, d, c ); \ + } while(0) + +#define stbir__x4_rem2( ofs ) \ + do { \ + stbir__simdf d; \ + stbir__simdf_load2z( c, h0 + (ofs) ); \ + stbir__simdf_load2( d, d0 + (ofs) ); \ + stbir__simdf_madd( a0, a0, d, c ); \ + stbir__simdf_load2z( c, h1 + (ofs) ); \ + stbir__simdf_load2( d, d1 + (ofs) ); \ + stbir__simdf_madd( a1, a1, d, c ); \ + stbir__simdf_load2z( c, h2 + (ofs) ); \ + stbir__simdf_load2( d, d2 + (ofs) ); \ + stbir__simdf_madd( a2, a2, d, c ); \ + stbir__simdf_load2z( c, h3 + (ofs) ); \ + stbir__simdf_load2( d, d3 + (ofs) ); \ + stbir__simdf_madd( a3, a3, d, c ); \ + } while(0) + +// widths of 4n+3 carry a zero after their last coefficient (STBIR__COEFFICIENT_PAD), so three is a full four +#define stbir__x4_rem3( ofs ) stbir__x4_cont( ofs ) + +// two coefficients: ( d0 d0+1 | d1 d1+1 ) * ( c0 c0+1 | c1 c1+1 ), twice, then one pairwise add gives the four outputs +#define stbir__x4_2taps() \ + do { \ + stbir__simdf_load2( c, h0 ); \ + stbir__simdf_load2hmerge( c, c, h1 ); \ + stbir__simdf_load2( a0, d0 ); \ + stbir__simdf_load2hmerge( a0, a0, d1 ); \ + stbir__simdf_mult( a0, a0, c ); \ + stbir__simdf_load2( a2, h2 ); \ + stbir__simdf_load2hmerge( a2, a2, h3 ); \ + stbir__simdf_load2( a3, d2 ); \ + stbir__simdf_load2hmerge( a3, a3, d3 ); \ + stbir__simdf_mult( a1, a3, a2 ); \ + stbir__simdf_hadd2( a0, a0, a1 ); \ + } while(0) + +// the same association as the per pixel path's reduction, ( l0 + l2 ) + ( l1 + l3 ), so the results match it bit for bit +#define stbir__x4_finish() \ + do { \ + stbir__simdf_addhalves2( a0, a0, a1 ); \ + stbir__simdf_addhalves2( a2, a2, a3 ); \ + stbir__simdf_hadd2( a0, a0, a2 ); \ + } while(0) + +// the three coefficient per pixel path reduces as ( l0 + l1 ) + l2, which the pairwise tree reproduces (lane 3 is the zero pad) +#define stbir__x4_finish3() \ + do { \ + stbir__simdf_hadd2( a0, a0, a1 ); \ + stbir__simdf_hadd2( a2, a2, a3 ); \ + stbir__simdf_hadd2( a0, a0, a2 ); \ + } while(0) + +#define stbir__x4_store() \ + do { \ + stbir__simdf_store( output, a0 ); \ + decode_buffer = d3 + e3->advance; \ + column = (stbir__column_entry const *)( (char const *)e3 + entry_stride ); \ + output += 4; \ + } while(0) + +// polyphase period 1 for 1 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf P0; stbir__simdf_load( P0, hc ); +#define stbir__period1_setup_5() stbir__simdf P0, PR; stbir__simdf_load( P0, hc ); stbir__simdf_load1z( PR, hc + 4 ); +#define stbir__period1_setup_8() stbir__simdf P0, P1; stbir__simdf_load( P0, hc ); stbir__simdf_load( P1, hc + 4 ); +#define stbir__period1_setup_9() stbir__simdf P0, P1, PR; stbir__simdf_load( P0, hc ); stbir__simdf_load( P1, hc + 4 ); stbir__simdf_load1z( PR, hc + 8 ); +#define stbir__period1_setup_11() stbir__simdf P0, P1, PR3; stbir__simdf_load( P0, hc ); stbir__simdf_load( P1, hc + 4 ); stbir__simdf_load( PR3, hc + 8 ); // the fourth of PR3 is the 4n+3 zero pad +#define stbir__period1_setup_12() stbir__simdf P0, P1, P2; stbir__simdf_load( P0, hc ); stbir__simdf_load( P1, hc + 4 ); stbir__simdf_load( P2, hc + 8 ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1z( PR, hc + 12 ); +#define stbir__period1_setup_16() stbir__simdf P3; stbir__period1_setup_12() stbir__simdf_load( P3, hc + 12 ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1z( PR, hc + 16 ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf tot, c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_mult_mem( tot, P0, decode ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf_madd_mem( tot, tot, P1, decode + 4 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf_madd_mem( tot, tot, P2, decode + 8 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf_madd_mem( tot, tot, P3, decode + 12 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + do { \ + stbir__simdf d; \ + stbir__simdf_load1( d, decode + (ofs) ); \ + stbir__simdf_madd( tot, tot, d, PR ); \ + } while(0) +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot, tot, PR3, decode + (ofs) ); + +#define stbir__period1_store_output() \ + stbir__simdf_0123to2301( c, tot ); \ + stbir__simdf_add( tot, tot, c ); \ + stbir__simdf_0123to1230( c, tot ); \ + stbir__simdf_add1( tot, tot, c ); \ + stbir__simdf_store1( output, tot ); \ + decode_buffer += advance; \ + output += 1; + +// the four outputs' decode pointers are one base plus multiples of the (constant) advance +#define stbir__period1_x4_setup() \ + float const * d0 = decode_buffer; \ + int const dofs1 = advance, dofs2 = advance * 2, dofs3 = advance * 3; \ + stbir__simdf a0, a1, a2, a3; + +#define stbir__period1_x4_start() \ + do { \ + stbir__simdf_mult_mem( a0, P0, d0 ); \ + stbir__simdf_mult_mem( a1, P0, d0 + dofs1 ); \ + stbir__simdf_mult_mem( a2, P0, d0 + dofs2 ); \ + stbir__simdf_mult_mem( a3, P0, d0 + dofs3 ); \ + } while(0) + +// the next four coefficients, for each of the four outputs +#define stbir__period1_x4_next( P, ofs ) \ + do { \ + stbir__simdf_madd_mem( a0, a0, P, d0 + (ofs) ); \ + stbir__simdf_madd_mem( a1, a1, P, d0 + dofs1 + (ofs) ); \ + stbir__simdf_madd_mem( a2, a2, P, d0 + dofs2 + (ofs) ); \ + stbir__simdf_madd_mem( a3, a3, P, d0 + dofs3 + (ofs) ); \ + } while(0) +#define stbir__period1_x4_continue() stbir__period1_x4_next( P1, 4 ) +#define stbir__period1_x4_continue2() stbir__period1_x4_next( P2, 8 ) +#define stbir__period1_x4_continue3() stbir__period1_x4_next( P3, 12 ) +#define stbir__period1_x4_remnant3( ofs ) stbir__period1_x4_next( PR3, ofs ) + +#define stbir__period1_x4_remnant( ofs ) \ + do { \ + stbir__simdf d; \ + stbir__simdf_load1( d, d0 + (ofs) ); \ + stbir__simdf_madd( a0, a0, d, PR ); \ + stbir__simdf_load1( d, d0 + dofs1 + (ofs) ); \ + stbir__simdf_madd( a1, a1, d, PR ); \ + stbir__simdf_load1( d, d0 + dofs2 + (ofs) ); \ + stbir__simdf_madd( a2, a2, d, PR ); \ + stbir__simdf_load1( d, d0 + dofs3 + (ofs) ); \ + stbir__simdf_madd( a3, a3, d, PR ); \ + } while(0) + +#define stbir__period1_x4_store() \ + do { \ + stbir__simdf_addhalves2( a0, a0, a1 ); \ + stbir__simdf_addhalves2( a2, a2, a3 ); \ + stbir__simdf_hadd2( a0, a0, a2 ); \ + stbir__simdf_store( output, a0 ); \ + decode_buffer = d0 + dofs3 + advance; \ + output += 4; \ + } while(0) + #define stbir__store_output() \ stbir__simdf_0123to2301( c, tot ); \ stbir__simdf_add( tot, tot, c ); \ stbir__simdf_0123to1230( c, tot ); \ stbir__simdf_add1( tot, tot, c ); \ stbir__simdf_store1( output, tot ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 1; #else @@ -4799,8 +5664,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output_tiny() \ output[0] = tot; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 1; #define stbir__4_coeff_start() \ @@ -4830,8 +5694,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output() \ output[0] = (tot0+tot2)+(tot1+tot3); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 1; #endif @@ -4875,8 +5738,52 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_0123to2301( c, tot ); \ stbir__simdf_add( tot, tot, c ); \ stbir__simdf_store2( output, tot ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ + output += 2; + +// polyphase period 1 for 2 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf P0a, P0b, cs; stbir__simdf_load( cs, hc ); stbir__simdf_0123to0011( P0a, cs ); stbir__simdf_0123to2233( P0b, cs ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1z( cs, hc + 4 ); stbir__simdf_0123to0011( PR, cs ); +#define stbir__period1_setup_8() stbir__simdf P1a, P1b; stbir__period1_setup_4() stbir__simdf_load( cs, hc + 4 ); stbir__simdf_0123to0011( P1a, cs ); stbir__simdf_0123to2233( P1b, cs ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1z( cs, hc + 8 ); stbir__simdf_0123to0011( PR, cs ); +#define stbir__period1_setup_11() stbir__simdf PR3a, PR3b; stbir__period1_setup_8() stbir__simdf_load( cs, hc + 8 ); stbir__simdf_0123to0011( PR3a, cs ); stbir__simdf_0123to2222( PR3b, cs ); +#define stbir__period1_setup_12() stbir__simdf P2a, P2b; stbir__period1_setup_8() stbir__simdf_load( cs, hc + 8 ); stbir__simdf_0123to0011( P2a, cs ); stbir__simdf_0123to2233( P2b, cs ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1z( cs, hc + 12 ); stbir__simdf_0123to0011( PR, cs ); +#define stbir__period1_setup_16() stbir__simdf P3a, P3b; stbir__period1_setup_12() stbir__simdf_load( cs, hc + 12 ); stbir__simdf_0123to0011( P3a, cs ); stbir__simdf_0123to2233( P3b, cs ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1z( cs, hc + 16 ); stbir__simdf_0123to0011( PR, cs ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf tot0, tot1; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_mult_mem( tot0, P0a, decode ); \ + stbir__simdf_mult_mem( tot1, P0b, decode + 4 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf_madd_mem( tot0, tot0, P1a, decode + 8 ); \ + stbir__simdf_madd_mem( tot1, tot1, P1b, decode + 12 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf_madd_mem( tot0, tot0, P2a, decode + 16 ); \ + stbir__simdf_madd_mem( tot1, tot1, P2b, decode + 20 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf_madd_mem( tot0, tot0, P3a, decode + 24 ); \ + stbir__simdf_madd_mem( tot1, tot1, P3b, decode + 28 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_load2( d, decode + (ofs) * 2 ); \ + stbir__simdf_madd( tot0, tot0, d, PR ); } +#define stbir__period1_3_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_madd_mem( tot0, tot0, PR3a, decode + (ofs) * 2 ); \ + stbir__simdf_load2z( d, decode + (ofs) * 2 + 4 ); \ + stbir__simdf_madd( tot1, tot1, d, PR3b ); } + +#define stbir__period1_store_output() \ + stbir__simdf_add( tot0, tot0, tot1 ); \ + stbir__simdf_0123to2301( tot1, tot0 ); \ + stbir__simdf_add( tot0, tot0, tot1 ); \ + stbir__simdf_store2( output, tot0 ); \ + decode_buffer += advance; \ output += 2; #ifdef STBIR_SIMD8 @@ -4922,8 +5829,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_0123to2301( d, t ); \ stbir__simdf_add( t, t, d ); \ stbir__simdf_store2( output, t ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 2; } #else @@ -4971,8 +5877,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_0123to2301( c, tot0 ); \ stbir__simdf_add( tot0, tot0, c ); \ stbir__simdf_store2( output, tot0 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 2; #endif @@ -5010,8 +5915,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output_tiny() \ output[0] = tota; \ output[1] = totb; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 2; #define stbir__4_coeff_start() \ @@ -5070,8 +5974,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output() \ output[0] = (tota0+tota2)+(tota1+tota3); \ output[1] = (totb0+totb2)+(totb1+totb3); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 2; #endif @@ -5123,8 +6026,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_store2( output, tot ); \ stbir__simdf_0123to2301( tot, tot ); \ stbir__simdf_store1( output+2, tot ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 3; #ifdef STBIR_SIMD8 @@ -5166,12 +6068,60 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf8_0123to2222( t, cs ); \ stbir__simdf8_madd_mem4( tot1, tot1, t, decode+(ofs)*3 + 6 - 1 ); +// polyphase period 1 for 3 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf8 P01, P23, cs; stbir__simdf8_load4b( cs, hc ); stbir__simdf8_0123to00001111( P01, cs ); stbir__simdf8_0123to22223333( P23, cs ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1rep4( PR, hc + 4 ); +#define stbir__period1_setup_8() stbir__simdf8 P45, P67; stbir__period1_setup_4() stbir__simdf8_load4b( cs, hc + 4 ); stbir__simdf8_0123to00001111( P45, cs ); stbir__simdf8_0123to22223333( P67, cs ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1rep4( PR, hc + 8 ); +#define stbir__period1_setup_11() stbir__simdf8 P89; stbir__simdf PR; stbir__period1_setup_8() stbir__simdf8_load4b( cs, hc + 8 ); stbir__simdf8_0123to00001111( P89, cs ); stbir__simdf8_0123to2222( PR, cs ); +#define stbir__period1_setup_12() stbir__simdf8 P89, PAB; stbir__period1_setup_8() stbir__simdf8_load4b( cs, hc + 8 ); stbir__simdf8_0123to00001111( P89, cs ); stbir__simdf8_0123to22223333( PAB, cs ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1rep4( PR, hc + 12 ); +#define stbir__period1_setup_16() stbir__simdf8 PCD, PEF; stbir__period1_setup_12() stbir__simdf8_load4b( cs, hc + 12 ); stbir__simdf8_0123to00001111( PCD, cs ); stbir__simdf8_0123to22223333( PEF, cs ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1rep4( PR, hc + 16 ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf8 tot0, tot1; stbir__simdf t; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_mult_mem( tot0, P01, decode - 1 ); \ + stbir__simdf8_mult_mem( tot1, P23, decode + 6 - 1 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf8_madd_mem( tot0, tot0, P45, decode + 12 - 1 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P67, decode + 18 - 1 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf8_madd_mem( tot0, tot0, P89, decode + 24 - 1 ); \ + stbir__simdf8_madd_mem( tot1, tot1, PAB, decode + 30 - 1 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf8_madd_mem( tot0, tot0, PCD, decode + 36 - 1 ); \ + stbir__simdf8_madd_mem( tot1, tot1, PEF, decode + 42 - 1 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem4( tot0, tot0, PR, decode + (ofs) * 3 - 1 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem( tot0, tot0, P89, decode + (ofs) * 3 - 1 ); \ + stbir__simdf8_madd_mem4( tot1, tot1, PR, decode + (ofs) * 3 + 6 - 1 ); + +#define stbir__period1_store_output() \ + stbir__simdf8_add( tot0, tot0, tot1 ); \ + stbir__simdf_0123to1230( t, stbir__if_simdf8_cast_to_simdf4( tot0 ) ); \ + stbir__simdf8_add4halves( t, t, tot0 ); \ + decode_buffer += advance; \ + output += 3; \ + if ( output < output_end ) \ + { \ + stbir__simdf_store( output-3, t ); \ + continue; \ + } \ + { stbir__simdf tt; stbir__simdf_0123to2301( tt, t ); \ + stbir__simdf_store2( output-3, t ); \ + stbir__simdf_store1( output+2-3, tt ); } \ + break; + #define stbir__store_output() \ stbir__simdf8_add( tot0, tot0, tot1 ); \ stbir__simdf_0123to1230( t, stbir__if_simdf8_cast_to_simdf4( tot0 ) ); \ stbir__simdf8_add4halves( t, t, tot0 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 3; \ if ( output < output_end ) \ { \ @@ -5235,6 +6185,65 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_load1z( d, decode+(ofs)*3+8 ); \ stbir__simdf_madd( tot2, tot2, c, d ); } +// polyphase period 1 for 3 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf P0a, P0b, P0c, cs; stbir__simdf_load( cs, hc ); stbir__simdf_0123to0001( P0a, cs ); stbir__simdf_0123to1122( P0b, cs ); stbir__simdf_0123to2333( P0c, cs ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1z( cs, hc + 4 ); stbir__simdf_0123to0001( PR, cs ); +#define stbir__period1_setup_8() stbir__simdf P1a, P1b, P1c; stbir__period1_setup_4() stbir__simdf_load( cs, hc + 4 ); stbir__simdf_0123to0001( P1a, cs ); stbir__simdf_0123to1122( P1b, cs ); stbir__simdf_0123to2333( P1c, cs ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1z( cs, hc + 8 ); stbir__simdf_0123to0001( PR, cs ); +#define stbir__period1_setup_11() stbir__simdf PR3a, PR3b, PR3c; stbir__period1_setup_8() stbir__simdf_load( cs, hc + 8 ); stbir__simdf_0123to0001( PR3a, cs ); stbir__simdf_0123to1122( PR3b, cs ); stbir__simdf_0123to2222( PR3c, cs ); +#define stbir__period1_setup_12() stbir__simdf P2a, P2b, P2c; stbir__period1_setup_8() stbir__simdf_load( cs, hc + 8 ); stbir__simdf_0123to0001( P2a, cs ); stbir__simdf_0123to1122( P2b, cs ); stbir__simdf_0123to2333( P2c, cs ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1z( cs, hc + 12 ); stbir__simdf_0123to0001( PR, cs ); +#define stbir__period1_setup_16() stbir__simdf P3a, P3b, P3c; stbir__period1_setup_12() stbir__simdf_load( cs, hc + 12 ); stbir__simdf_0123to0001( P3a, cs ); stbir__simdf_0123to1122( P3b, cs ); stbir__simdf_0123to2333( P3c, cs ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1z( cs, hc + 16 ); stbir__simdf_0123to0001( PR, cs ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf tot0, tot1, tot2, c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_mult_mem( tot0, P0a, decode ); \ + stbir__simdf_mult_mem( tot1, P0b, decode + 4 ); \ + stbir__simdf_mult_mem( tot2, P0c, decode + 8 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf_madd_mem( tot0, tot0, P1a, decode + 12 ); \ + stbir__simdf_madd_mem( tot1, tot1, P1b, decode + 16 ); \ + stbir__simdf_madd_mem( tot2, tot2, P1c, decode + 20 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf_madd_mem( tot0, tot0, P2a, decode + 24 ); \ + stbir__simdf_madd_mem( tot1, tot1, P2b, decode + 28 ); \ + stbir__simdf_madd_mem( tot2, tot2, P2c, decode + 32 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf_madd_mem( tot0, tot0, P3a, decode + 36 ); \ + stbir__simdf_madd_mem( tot1, tot1, P3b, decode + 40 ); \ + stbir__simdf_madd_mem( tot2, tot2, P3c, decode + 44 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot0, tot0, PR, decode + (ofs) * 3 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_madd_mem( tot0, tot0, PR3a, decode + (ofs) * 3 ); \ + stbir__simdf_madd_mem( tot1, tot1, PR3b, decode + (ofs) * 3 + 4 ); \ + stbir__simdf_load1z( d, decode + (ofs) * 3 + 8 ); \ + stbir__simdf_madd( tot2, tot2, PR3c, d ); } + +#define stbir__period1_store_output() \ + stbir__simdf_0123ABCDto3ABx( c, tot0, tot1 ); \ + stbir__simdf_0123ABCDto23Ax( cs, tot1, tot2 ); \ + stbir__simdf_0123to1230( tot2, tot2 ); \ + stbir__simdf_add( tot0, tot0, cs ); \ + stbir__simdf_add( c, c, tot2 ); \ + stbir__simdf_add( tot0, tot0, c ); \ + decode_buffer += advance; \ + output += 3; \ + if ( output < output_end ) \ + { \ + stbir__simdf_store( output-3, tot0 ); \ + continue; \ + } \ + stbir__simdf_0123to2301( tot1, tot0 ); \ + stbir__simdf_store2( output-3, tot0 ); \ + stbir__simdf_store1( output+2-3, tot1 ); \ + break; + #define stbir__store_output() \ stbir__simdf_0123ABCDto3ABx( c, tot0, tot1 ); \ stbir__simdf_0123ABCDto23Ax( cs, tot1, tot2 ); \ @@ -5242,8 +6251,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_add( tot0, tot0, cs ); \ stbir__simdf_add( c, c, tot2 ); \ stbir__simdf_add( tot0, tot0, c ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 3; \ if ( output < output_end ) \ { \ @@ -5296,8 +6304,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[0] = tot0; \ output[1] = tot1; \ output[2] = tot2; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 3; #define stbir__4_coeff_start() \ @@ -5371,8 +6378,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[0] = (tota0+totc0)+(totb0+totd0); \ output[1] = (tota1+totc1)+(totb1+totd1); \ output[2] = (tota2+totc2)+(totb2+totd2); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 3; #endif @@ -5415,8 +6421,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output_tiny() \ stbir__simdf_store( output, tot ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 4; #ifdef STBIR_SIMD8 @@ -5457,11 +6462,49 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf8_0123to2222( t, cs ); \ stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*4+8 ); +// polyphase period 1 for 4 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf8 P01, P23, cs; stbir__simdf8_load4b( cs, hc ); stbir__simdf8_0123to00001111( P01, cs ); stbir__simdf8_0123to22223333( P23, cs ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1rep4( PR, hc + 4 ); +#define stbir__period1_setup_8() stbir__simdf8 P45, P67; stbir__period1_setup_4() stbir__simdf8_load4b( cs, hc + 4 ); stbir__simdf8_0123to00001111( P45, cs ); stbir__simdf8_0123to22223333( P67, cs ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1rep4( PR, hc + 8 ); +#define stbir__period1_setup_11() stbir__simdf8 P89; stbir__simdf PR; stbir__period1_setup_8() stbir__simdf8_load4b( cs, hc + 8 ); stbir__simdf8_0123to00001111( P89, cs ); stbir__simdf8_0123to2222( PR, cs ); +#define stbir__period1_setup_12() stbir__simdf8 P89, PAB; stbir__period1_setup_8() stbir__simdf8_load4b( cs, hc + 8 ); stbir__simdf8_0123to00001111( P89, cs ); stbir__simdf8_0123to22223333( PAB, cs ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1rep4( PR, hc + 12 ); +#define stbir__period1_setup_16() stbir__simdf8 PCD, PEF; stbir__period1_setup_12() stbir__simdf8_load4b( cs, hc + 12 ); stbir__simdf8_0123to00001111( PCD, cs ); stbir__simdf8_0123to22223333( PEF, cs ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1rep4( PR, hc + 16 ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf8 tot0; stbir__simdf t; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_mult_mem( tot0, P01, decode ); \ + stbir__simdf8_madd_mem( tot0, tot0, P23, decode + 8 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf8_madd_mem( tot0, tot0, P45, decode + 16 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P67, decode + 24 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf8_madd_mem( tot0, tot0, P89, decode + 32 ); \ + stbir__simdf8_madd_mem( tot0, tot0, PAB, decode + 40 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf8_madd_mem( tot0, tot0, PCD, decode + 48 ); \ + stbir__simdf8_madd_mem( tot0, tot0, PEF, decode + 56 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem4( tot0, tot0, PR, decode + (ofs) * 4 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem( tot0, tot0, P89, decode + (ofs) * 4 ); \ + stbir__simdf8_madd_mem4( tot0, tot0, PR, decode + (ofs) * 4 + 8 ); + +#define stbir__period1_store_output() \ + stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \ + stbir__simdf_store( output, t ); \ + decode_buffer += advance; \ + output += 4; + #define stbir__store_output() \ stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \ stbir__simdf_store( output, t ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 4; #else @@ -5515,11 +6558,58 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_0123to2222( c, cs ); \ stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 ); +// polyphase period 1 for 4 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf P0, P1, P2, P3; stbir__simdf_load1frep4( P0, hc[0] ); stbir__simdf_load1frep4( P1, hc[1] ); stbir__simdf_load1frep4( P2, hc[2] ); stbir__simdf_load1frep4( P3, hc[3] ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1frep4( PR, hc[4] ); +#define stbir__period1_setup_8() stbir__simdf P4, P5, P6, P7; stbir__period1_setup_4() stbir__simdf_load1frep4( P4, hc[4] ); stbir__simdf_load1frep4( P5, hc[5] ); stbir__simdf_load1frep4( P6, hc[6] ); stbir__simdf_load1frep4( P7, hc[7] ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1frep4( PR, hc[8] ); +#define stbir__period1_setup_11() stbir__simdf P8, P9, P10; stbir__period1_setup_8() stbir__simdf_load1frep4( P8, hc[8] ); stbir__simdf_load1frep4( P9, hc[9] ); stbir__simdf_load1frep4( P10, hc[10] ); +#define stbir__period1_setup_12() stbir__simdf P8, P9, P10, P11; stbir__period1_setup_8() stbir__simdf_load1frep4( P8, hc[8] ); stbir__simdf_load1frep4( P9, hc[9] ); stbir__simdf_load1frep4( P10, hc[10] ); stbir__simdf_load1frep4( P11, hc[11] ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1frep4( PR, hc[12] ); +#define stbir__period1_setup_16() stbir__simdf P12, P13, P14, P15; stbir__period1_setup_12() stbir__simdf_load1frep4( P12, hc[12] ); stbir__simdf_load1frep4( P13, hc[13] ); stbir__simdf_load1frep4( P14, hc[14] ); stbir__simdf_load1frep4( P15, hc[15] ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1frep4( PR, hc[16] ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf tot0, tot1; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_mult_mem( tot0, P0, decode ); \ + stbir__simdf_mult_mem( tot1, P1, decode + 4 ); \ + stbir__simdf_madd_mem( tot0, tot0, P2, decode + 8 ); \ + stbir__simdf_madd_mem( tot1, tot1, P3, decode + 12 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf_madd_mem( tot0, tot0, P4, decode + 16 ); \ + stbir__simdf_madd_mem( tot1, tot1, P5, decode + 20 ); \ + stbir__simdf_madd_mem( tot0, tot0, P6, decode + 24 ); \ + stbir__simdf_madd_mem( tot1, tot1, P7, decode + 28 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf_madd_mem( tot0, tot0, P8, decode + 32 ); \ + stbir__simdf_madd_mem( tot1, tot1, P9, decode + 36 ); \ + stbir__simdf_madd_mem( tot0, tot0, P10, decode + 40 ); \ + stbir__simdf_madd_mem( tot1, tot1, P11, decode + 44 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf_madd_mem( tot0, tot0, P12, decode + 48 ); \ + stbir__simdf_madd_mem( tot1, tot1, P13, decode + 52 ); \ + stbir__simdf_madd_mem( tot0, tot0, P14, decode + 56 ); \ + stbir__simdf_madd_mem( tot1, tot1, P15, decode + 60 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot0, tot0, PR, decode + (ofs) * 4 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot0, tot0, P8, decode + (ofs) * 4 ); \ + stbir__simdf_madd_mem( tot1, tot1, P9, decode + (ofs) * 4 + 4 ); \ + stbir__simdf_madd_mem( tot0, tot0, P10, decode + (ofs) * 4 + 8 ); + +#define stbir__period1_store_output() \ + stbir__simdf_add( tot0, tot0, tot1 ); \ + stbir__simdf_store( output, tot0 ); \ + decode_buffer += advance; \ + output += 4; + #define stbir__store_output() \ stbir__simdf_add( tot0, tot0, tot1 ); \ stbir__simdf_store( output, tot0 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 4; #endif @@ -5573,8 +6663,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[1] = p1; \ output[2] = p2; \ output[3] = p3; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 4; #define stbir__4_coeff_start() \ @@ -5668,8 +6757,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[1] = x1 + y1; \ output[2] = x2 + y2; \ output[3] = x3 + y3; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 4; #endif @@ -5679,7 +6767,6 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #include STBIR__HEADER_FILENAME - //================= // Do 7 channel horizontal routines @@ -5721,8 +6808,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float #define stbir__store_output_tiny() \ stbir__simdf_store( output+3, tot1 ); \ stbir__simdf_store( output, tot0 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 7; #ifdef STBIR_SIMD8 @@ -5774,10 +6860,64 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf8_0123to22222222( c, cs ); \ stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); +// polyphase period 1 for 7 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf8 P0, P1, P2, P3; stbir__simdf8_load1b( P0, hc ); stbir__simdf8_load1b( P1, hc + 1 ); stbir__simdf8_load1b( P2, hc + 2 ); stbir__simdf8_load1b( P3, hc + 3 ); +#define stbir__period1_setup_5() stbir__simdf8 PR; stbir__period1_setup_4() stbir__simdf8_load1b( PR, hc + 4 ); +#define stbir__period1_setup_8() stbir__simdf8 P4, P5, P6, P7; stbir__period1_setup_4() stbir__simdf8_load1b( P4, hc + 4 ); stbir__simdf8_load1b( P5, hc + 5 ); stbir__simdf8_load1b( P6, hc + 6 ); stbir__simdf8_load1b( P7, hc + 7 ); +#define stbir__period1_setup_9() stbir__simdf8 PR; stbir__period1_setup_8() stbir__simdf8_load1b( PR, hc + 8 ); +#define stbir__period1_setup_11() stbir__simdf8 P8, P9, P10; stbir__period1_setup_8() stbir__simdf8_load1b( P8, hc + 8 ); stbir__simdf8_load1b( P9, hc + 9 ); stbir__simdf8_load1b( P10, hc + 10 ); +#define stbir__period1_setup_12() stbir__simdf8 P8, P9, P10, P11; stbir__period1_setup_8() stbir__simdf8_load1b( P8, hc + 8 ); stbir__simdf8_load1b( P9, hc + 9 ); stbir__simdf8_load1b( P10, hc + 10 ); stbir__simdf8_load1b( P11, hc + 11 ); +#define stbir__period1_setup_13() stbir__simdf8 PR; stbir__period1_setup_12() stbir__simdf8_load1b( PR, hc + 12 ); +#define stbir__period1_setup_16() stbir__simdf8 P12, P13, P14, P15; stbir__period1_setup_12() stbir__simdf8_load1b( P12, hc + 12 ); stbir__simdf8_load1b( P13, hc + 13 ); stbir__simdf8_load1b( P14, hc + 14 ); stbir__simdf8_load1b( P15, hc + 15 ); +#define stbir__period1_setup_17() stbir__simdf8 PR; stbir__period1_setup_16() stbir__simdf8_load1b( PR, hc + 16 ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf8 tot0, tot1; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_mult_mem( tot0, P0, decode ); \ + stbir__simdf8_mult_mem( tot1, P1, decode + 7 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P2, decode + 14 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P3, decode + 21 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf8_madd_mem( tot0, tot0, P4, decode + 28 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P5, decode + 35 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P6, decode + 42 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P7, decode + 49 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf8_madd_mem( tot0, tot0, P8, decode + 56 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P9, decode + 63 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P10, decode + 70 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P11, decode + 77 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf8_madd_mem( tot0, tot0, P12, decode + 84 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P13, decode + 91 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P14, decode + 98 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P15, decode + 105 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem( tot0, tot0, PR, decode + (ofs) * 7 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf8_madd_mem( tot0, tot0, P8, decode + (ofs) * 7 ); \ + stbir__simdf8_madd_mem( tot1, tot1, P9, decode + (ofs) * 7 + 7 ); \ + stbir__simdf8_madd_mem( tot0, tot0, P10, decode + (ofs) * 7 + 14 ); + +#define stbir__period1_store_output() \ + stbir__simdf8_add( tot0, tot0, tot1 ); \ + decode_buffer += advance; \ + output += 7; \ + if ( output < output_end ) \ + { \ + stbir__simdf8_store( output-7, tot0 ); \ + continue; \ + } \ + stbir__simdf_store( output-7+3, stbir__simdf_swiz(stbir__simdf8_gettop4(tot0),0,0,1,2) ); \ + stbir__simdf_store( output-7, stbir__if_simdf8_cast_to_simdf4(tot0) ); \ + break; + #define stbir__store_output() \ stbir__simdf8_add( tot0, tot0, tot1 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 7; \ if ( output < output_end ) \ { \ @@ -5853,13 +6993,82 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \ stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+17 ); +// polyphase period 1 for 7 channels (when have exact downsamples 2:1, 3:1, etc - common for mipmaps) +#define stbir__period1_setup_4() stbir__simdf P0, P1, P2, P3; stbir__simdf_load1frep4( P0, hc[0] ); stbir__simdf_load1frep4( P1, hc[1] ); stbir__simdf_load1frep4( P2, hc[2] ); stbir__simdf_load1frep4( P3, hc[3] ); +#define stbir__period1_setup_5() stbir__simdf PR; stbir__period1_setup_4() stbir__simdf_load1frep4( PR, hc[4] ); +#define stbir__period1_setup_8() stbir__simdf P4, P5, P6, P7; stbir__period1_setup_4() stbir__simdf_load1frep4( P4, hc[4] ); stbir__simdf_load1frep4( P5, hc[5] ); stbir__simdf_load1frep4( P6, hc[6] ); stbir__simdf_load1frep4( P7, hc[7] ); +#define stbir__period1_setup_9() stbir__simdf PR; stbir__period1_setup_8() stbir__simdf_load1frep4( PR, hc[8] ); +#define stbir__period1_setup_11() stbir__simdf P8, P9, P10; stbir__period1_setup_8() stbir__simdf_load1frep4( P8, hc[8] ); stbir__simdf_load1frep4( P9, hc[9] ); stbir__simdf_load1frep4( P10, hc[10] ); +#define stbir__period1_setup_12() stbir__simdf P8, P9, P10, P11; stbir__period1_setup_8() stbir__simdf_load1frep4( P8, hc[8] ); stbir__simdf_load1frep4( P9, hc[9] ); stbir__simdf_load1frep4( P10, hc[10] ); stbir__simdf_load1frep4( P11, hc[11] ); +#define stbir__period1_setup_13() stbir__simdf PR; stbir__period1_setup_12() stbir__simdf_load1frep4( PR, hc[12] ); +#define stbir__period1_setup_16() stbir__simdf P12, P13, P14, P15; stbir__period1_setup_12() stbir__simdf_load1frep4( P12, hc[12] ); stbir__simdf_load1frep4( P13, hc[13] ); stbir__simdf_load1frep4( P14, hc[14] ); stbir__simdf_load1frep4( P15, hc[15] ); +#define stbir__period1_setup_17() stbir__simdf PR; stbir__period1_setup_16() stbir__simdf_load1frep4( PR, hc[16] ); + +#define stbir__period1_4_coeff_start() \ + stbir__simdf tot0, tot1, tot2, tot3; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_mult_mem( tot0, P0, decode ); \ + stbir__simdf_mult_mem( tot1, P0, decode + 3 ); \ + stbir__simdf_mult_mem( tot2, P1, decode + 7 ); \ + stbir__simdf_mult_mem( tot3, P1, decode + 10 ); \ + stbir__simdf_madd_mem( tot0, tot0, P2, decode + 14 ); \ + stbir__simdf_madd_mem( tot1, tot1, P2, decode + 17 ); \ + stbir__simdf_madd_mem( tot2, tot2, P3, decode + 21 ); \ + stbir__simdf_madd_mem( tot3, tot3, P3, decode + 24 ); + +#define stbir__period1_4_coeff_continue() \ + stbir__simdf_madd_mem( tot0, tot0, P4, decode + 28 ); \ + stbir__simdf_madd_mem( tot1, tot1, P4, decode + 31 ); \ + stbir__simdf_madd_mem( tot2, tot2, P5, decode + 35 ); \ + stbir__simdf_madd_mem( tot3, tot3, P5, decode + 38 ); \ + stbir__simdf_madd_mem( tot0, tot0, P6, decode + 42 ); \ + stbir__simdf_madd_mem( tot1, tot1, P6, decode + 45 ); \ + stbir__simdf_madd_mem( tot2, tot2, P7, decode + 49 ); \ + stbir__simdf_madd_mem( tot3, tot3, P7, decode + 52 ); +#define stbir__period1_4_coeff_continue2() \ + stbir__simdf_madd_mem( tot0, tot0, P8, decode + 56 ); \ + stbir__simdf_madd_mem( tot1, tot1, P8, decode + 59 ); \ + stbir__simdf_madd_mem( tot2, tot2, P9, decode + 63 ); \ + stbir__simdf_madd_mem( tot3, tot3, P9, decode + 66 ); \ + stbir__simdf_madd_mem( tot0, tot0, P10, decode + 70 ); \ + stbir__simdf_madd_mem( tot1, tot1, P10, decode + 73 ); \ + stbir__simdf_madd_mem( tot2, tot2, P11, decode + 77 ); \ + stbir__simdf_madd_mem( tot3, tot3, P11, decode + 80 ); +#define stbir__period1_4_coeff_continue3() \ + stbir__simdf_madd_mem( tot0, tot0, P12, decode + 84 ); \ + stbir__simdf_madd_mem( tot1, tot1, P12, decode + 87 ); \ + stbir__simdf_madd_mem( tot2, tot2, P13, decode + 91 ); \ + stbir__simdf_madd_mem( tot3, tot3, P13, decode + 94 ); \ + stbir__simdf_madd_mem( tot0, tot0, P14, decode + 98 ); \ + stbir__simdf_madd_mem( tot1, tot1, P14, decode + 101 ); \ + stbir__simdf_madd_mem( tot2, tot2, P15, decode + 105 ); \ + stbir__simdf_madd_mem( tot3, tot3, P15, decode + 108 ); + +#define stbir__period1_1_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot0, tot0, PR, decode + (ofs) * 7 ); \ + stbir__simdf_madd_mem( tot1, tot1, PR, decode + (ofs) * 7 + 3 ); +#define stbir__period1_3_coeff_remnant( ofs ) \ + stbir__simdf_madd_mem( tot0, tot0, P8, decode + (ofs) * 7 ); \ + stbir__simdf_madd_mem( tot1, tot1, P8, decode + (ofs) * 7 + 3 ); \ + stbir__simdf_madd_mem( tot2, tot2, P9, decode + (ofs) * 7 + 7 ); \ + stbir__simdf_madd_mem( tot3, tot3, P9, decode + (ofs) * 7 + 10 ); \ + stbir__simdf_madd_mem( tot0, tot0, P10, decode + (ofs) * 7 + 14 ); \ + stbir__simdf_madd_mem( tot1, tot1, P10, decode + (ofs) * 7 + 17 ); + +#define stbir__period1_store_output() \ + stbir__simdf_add( tot0, tot0, tot2 ); \ + stbir__simdf_add( tot1, tot1, tot3 ); \ + stbir__simdf_store( output+3, tot1 ); \ + stbir__simdf_store( output, tot0 ); \ + decode_buffer += advance; \ + output += 7; + #define stbir__store_output() \ stbir__simdf_add( tot0, tot0, tot2 ); \ stbir__simdf_add( tot1, tot1, tot3 ); \ stbir__simdf_store( output+3, tot1 ); \ stbir__simdf_store( output, tot0 ); \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 7; #endif @@ -5931,8 +7140,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[4] = tot4; \ output[5] = tot5; \ output[6] = tot6; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 7; #define stbir__4_coeff_start() \ @@ -6071,8 +7279,7 @@ static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float output[4] = x4 + y4; \ output[5] = x5 + y5; \ output[6] = x6 + y6; \ - horizontal_coefficients += coefficient_width; \ - ++horizontal_contributors; \ + stbir__next_contributor(); \ output += 7; #endif @@ -6181,11 +7388,18 @@ static STBIR_VERTICAL_SCATTERFUNC * stbir__vertical_scatter_blends[ 8 ] = }; -static void stbir__encode_scanline( stbir__info const * stbir_info, void *output_buffer_data, float * encode_buffer, int row STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +// pointer to an output pixel, relative to the output region (offset_x/offset_y are already folded into output_data) +static void * stbir__output_pixel_ptr( stbir__info const * stbir_info, int x, int y ) { - int num_pixels = stbir_info->horizontal.scale_info.output_sub_size; + return ( (char*) stbir_info->output_data ) + (ptrdiff_t)( y * stbir_info->output_stride_bytes ) + ( (size_t)x * (size_t)stbir_info->channels * (size_t)stbir__type_size[ stbir_info->output_type ] ); +} + +static void stbir__encode_scanline( stbir__info const * stbir_info, float * encode_buffer, int row, stbir__column_slice const * slice STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +{ + int num_pixels = slice->width; int channels = stbir_info->channels; - int width_times_channels = num_pixels * channels; + ptrdiff_t width_times_channels = (ptrdiff_t)num_pixels * channels; + void * output_buffer_data = stbir__output_pixel_ptr( stbir_info, slice->x0, row ); void * output_buffer; // un-alpha weight if we need to @@ -6208,9 +7422,9 @@ static void stbir__encode_scanline( stbir__info const * stbir_info, void *output stbir_info->encode_pixels( output_buffer, width_times_channels, encode_buffer ); STBIR_PROFILE_END( encode ); - // if we have an output callback, call it to send the data + // if we have an output callback, call it to send the data (x and y are in full output image space, not region relative) if ( stbir_info->out_pixels_cb ) - stbir_info->out_pixels_cb( output_buffer, num_pixels, row, stbir_info->user_data ); + stbir_info->out_pixels_cb( output_buffer, num_pixels, stbir_info->offset_x + slice->x0, stbir_info->offset_y + row, stbir_info->user_data ); } @@ -6222,7 +7436,7 @@ static float* stbir__get_ring_buffer_entry(stbir__info const * stbir_info, stbir #ifdef STBIR__SEPARATE_ALLOCATIONS return split_info->ring_buffers[ index ]; #else - return (float*) ( ( (char*) split_info->ring_buffer ) + ( index * stbir_info->ring_buffer_length_bytes ) ); + return (float*) ( ( (char*) split_info->ring_buffer ) + ( (size_t) index * stbir_info->ring_buffer_length_bytes ) ); #endif } @@ -6233,25 +7447,78 @@ static float* stbir__get_ring_buffer_scanline(stbir__info const * stbir_info, st return stbir__get_ring_buffer_entry( stbir_info, split_info, ring_buffer_index ); } -static void stbir__resample_horizontal_gather(stbir__info const * stbir_info, float* output_buffer, float const * input_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +// the horizontal pass over one slice's output pixels. +static void stbir__resample_horizontal_gather(stbir__info const * stbir_info, float* output_buffer, float const * input_buffer, stbir__column_slice const * slice STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) { - float const * decode_buffer = input_buffer - ( stbir_info->scanline_extents.conservative.n0 * stbir_info->effective_channels ); + stbir__sampler const * horizontal = &stbir_info->horizontal; + int channels = stbir_info->effective_channels; + int point_copy = ( horizontal->filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( stbir__scale_is_copy( &horizontal->scale_info ) ); + int prologue = horizontal->shared_prologue; + int period = horizontal->shared_period; + int epilogue_start = horizontal->num_contributors - horizontal->shared_epilogue; + int x = slice->x0, x_end = slice->x0 + slice->width; + int n0 = slice->first_n0; STBIR_PROFILE_START( horizontal ); - if ( ( stbir_info->horizontal.filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( stbir_info->horizontal.scale_info.scale == 1.0f ) ) - STBIR_MEMCPY( output_buffer, input_buffer, stbir_info->horizontal.scale_info.output_sub_size * sizeof( float ) * stbir_info->effective_channels ); - else - stbir_info->horizontal_gather_channels( output_buffer, stbir_info->horizontal.scale_info.output_sub_size, decode_buffer, stbir_info->horizontal.contributors, stbir_info->horizontal.coefficients, stbir_info->horizontal.coefficient_width ); + while ( x < x_end ) + { + float const * decode = input_buffer + (ptrdiff_t)( n0 - slice->in_n0 ) * channels; // input_buffer holds input pixel in_n0 at index zero + int run_end = x_end; + int entry_stride = horizontal->entry_floats * (int) sizeof( float ); + + if ( ( period ) && ( x < epilogue_start ) ) + { + if ( period == 1 ) + { + // one set for the whole periodic region: the prologue is its own run, then the region is one run over that set + run_end = ( x < prologue ) ? prologue : epilogue_start; + if ( x >= prologue ) + entry_stride = 0; + } + else + { + // to the end of this pass over the block (from the prologue, the first pass), but not into the epilogue + int pass = ( x < prologue ) ? 0 : ( ( x - prologue ) / period ); + run_end = prologue + ( pass + 1 ) * period; + if ( run_end > epilogue_start ) + run_end = epilogue_start; + } + if ( run_end > x_end ) + run_end = x_end; + } + + if ( point_copy ) + STBIR_MEMCPY( output_buffer, decode, ( run_end - x ) * sizeof( float ) * channels ); + else if ( ( entry_stride == 0 ) && ( stbir_info->horizontal_gather_period1 ) ) + stbir_info->horizontal_gather_period1( output_buffer, run_end - x, decode, stbir__output_column_entry( horizontal, x ), 0 ); + else + stbir_info->horizontal_gather_channels( output_buffer, run_end - x, decode, stbir__output_column_entry( horizontal, x ), entry_stride ); + output_buffer += (ptrdiff_t)( run_end - x ) * channels; + + // where the next run starts reading: the epilogue, or a block boundary + if ( run_end == epilogue_start ) + n0 = horizontal->shared_epilogue_n0; + else if ( period ) + n0 = horizontal->shared_block_n0 + ( ( run_end - prologue ) / period ) * horizontal->shared_block_shift; + x = run_end; + } STBIR_PROFILE_END( horizontal ); } -static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n, int contrib_n0, int contrib_n1, float const * vertical_coefficients ) +// how many floats wide the buffer that the vertical pass works on is (decoded input pixels when vertical +// first, horizontally resampled output pixels otherwise) +static ptrdiff_t stbir__vertical_width_times_channels( stbir__info const * stbir_info, stbir__column_slice const * slice ) +{ + int width = ( stbir_info->vertical_first ) ? ( slice->in_n1 - slice->in_n0 + 1 ) : slice->width; + return (ptrdiff_t)stbir_info->effective_channels * width; +} + +static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n, int contrib_n0, int contrib_n1, float const * vertical_coefficients, stbir__column_slice const * slice ) { float* encode_buffer = split_info->vertical_buffer; float* decode_buffer = split_info->decode_buffer; int vertical_first = stbir_info->vertical_first; - int width = (vertical_first) ? ( stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1 ) : stbir_info->horizontal.scale_info.output_sub_size; - int width_times_channels = stbir_info->effective_channels * width; + ptrdiff_t width_times_channels = stbir__vertical_width_times_channels( stbir_info, slice ); STBIR_ASSERT( stbir_info->vertical.is_gather ); @@ -6262,7 +7529,9 @@ static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbi STBIR_ASSERT( total > 0 ); do { float const * inputs[8]; - int i, cnt = total; if ( cnt > 8 ) cnt = 8; + int i, cnt = total; + if ( cnt > 8 ) + cnt = 8; for( i = 0 ; i < cnt ; i++ ) inputs[ i ] = stbir__get_ring_buffer_scanline(stbir_info, split_info, k+i+contrib_n0 ); @@ -6279,20 +7548,19 @@ static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbi // Now resample the gathered vertical data in the horizontal axis into the encode buffer decode_buffer[ width_times_channels ] = 0.0f; // clear two over for horizontals with a remnant of 3 decode_buffer[ width_times_channels+1 ] = 0.0f; - stbir__resample_horizontal_gather(stbir_info, encode_buffer, decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__resample_horizontal_gather(stbir_info, encode_buffer, decode_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); } - stbir__encode_scanline( stbir_info, ( (char *) stbir_info->output_data ) + ((size_t)n * (size_t)stbir_info->output_stride_bytes), - encode_buffer, n STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__encode_scanline( stbir_info, encode_buffer, n, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); } -static void stbir__decode_and_resample_for_vertical_gather_loop(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n) +static void stbir__decode_and_resample_for_vertical_gather_loop(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n, stbir__column_slice const * slice) { int ring_buffer_index; float* ring_buffer; // Decode the nth scanline from the source image into the decode buffer. - stbir__decode_scanline( stbir_info, n, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__decode_scanline( stbir_info, n, split_info->decode_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // update new end scanline split_info->ring_buffer_last_scanline = n; @@ -6302,36 +7570,70 @@ static void stbir__decode_and_resample_for_vertical_gather_loop(stbir__info cons ring_buffer = stbir__get_ring_buffer_entry(stbir_info, split_info, ring_buffer_index); // Now resample it into the ring buffer. - stbir__resample_horizontal_gather( stbir_info, ring_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__resample_horizontal_gather( stbir_info, ring_buffer, split_info->decode_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // Now it's sitting in the ring buffer ready to be used as source for the vertical sampling. } -static void stbir__vertical_gather_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count ) +// A trimmed zero coefficient can make a row's range start later than the next row's ((4,4) then (3,6)), and the ring only +// loads forward. So look ahead up to max_look rows from y for the lowest first scanline; *out_row is which row had it. +static int stbir__lowest_scanline_needed( stbir__sampler const * vertical, int y, int max_look, int * out_row ) { - int y, start_output_y, end_output_y; - stbir__contributors* vertical_contributors = stbir_info->vertical.contributors; - float const * vertical_coefficients = stbir_info->vertical.coefficients; + int start_n0, lowest_n0, n0, n1, j; - STBIR_ASSERT( stbir_info->vertical.is_gather ); + stbir__output_range( vertical, y, &start_n0, &n1 ); + lowest_n0 = start_n0; + *out_row = 0; + for (j = 1; j <= max_look; j++) + { + stbir__output_range( vertical, y + j, &n0, &n1 ); + if ( n0 > start_n0 ) + break; + if ( n0 < lowest_n0 ) + { + lowest_n0 = n0; + *out_row = j; + } + } + return lowest_n0; +} + +static void stbir__vertical_gather_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count, stbir__column_slice const * slice ) +{ + stbir__sampler const * vertical = &stbir_info->vertical; + int y, start_output_y, end_output_y; + + STBIR_ASSERT( vertical->is_gather ); start_output_y = split_info->start_output_y; end_output_y = split_info[split_count-1].end_output_y; - vertical_contributors += start_output_y; - vertical_coefficients += start_output_y * stbir_info->vertical.coefficient_width; - - // initialize the ring buffer for gathering - split_info->ring_buffer_begin_index = 0; - split_info->ring_buffer_first_scanline = vertical_contributors->n0; - split_info->ring_buffer_last_scanline = split_info->ring_buffer_first_scanline - 1; // means "empty" + // initialize the ring buffer for gathering: start it at the lowest scanline the first rows need + { + int look = vertical->filter_pixel_margin * 3, unused; + if ( look > ( end_output_y - start_output_y - 1 ) ) + look = end_output_y - start_output_y - 1; + split_info->ring_buffer_begin_index = 0; + split_info->ring_buffer_first_scanline = stbir__lowest_scanline_needed( vertical, start_output_y, look, &unused ); + split_info->ring_buffer_last_scanline = split_info->ring_buffer_first_scanline - 1; // means "empty" + } for (y = start_output_y; y < end_output_y; y++) { int in_first_scanline, in_last_scanline; + float const * vertical_coefficients = stbir__output_coefficients( vertical, y ); - in_first_scanline = vertical_contributors->n0; - in_last_scanline = vertical_contributors->n1; + stbir__output_range( vertical, y, &in_first_scanline, &in_last_scanline ); + + // rows with a zero weight at either end aren't worth decoding (the pack's shifted sets when the vertical shares the + // horizontal's tables, kernels that are exactly zero at their support's edge) + while ( ( in_first_scanline < in_last_scanline ) && ( vertical_coefficients[0] == 0.0f ) ) + { + ++in_first_scanline; + ++vertical_coefficients; + } + while ( ( in_last_scanline > in_first_scanline ) && ( vertical_coefficients[ in_last_scanline - in_first_scanline ] == 0.0f ) ) + --in_last_scanline; // make sure the indexing hasn't broken STBIR_ASSERT( in_first_scanline >= split_info->ring_buffer_first_scanline ); @@ -6352,32 +7654,29 @@ static void stbir__vertical_gather_loop( stbir__info const * stbir_info, stbir__ { float * ring_buffer = stbir__get_ring_buffer_scanline( stbir_info, split_info, ++split_info->ring_buffer_last_scanline ); // Decode the nth scanline from the source image into the decode buffer. - stbir__decode_scanline( stbir_info, split_info->ring_buffer_last_scanline, ring_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__decode_scanline( stbir_info, split_info->ring_buffer_last_scanline, ring_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); } else { - stbir__decode_and_resample_for_vertical_gather_loop(stbir_info, split_info, split_info->ring_buffer_last_scanline + 1); + stbir__decode_and_resample_for_vertical_gather_loop(stbir_info, split_info, split_info->ring_buffer_last_scanline + 1, slice); } } // Now all buffers should be ready to write a row of vertical sampling, so do it. - stbir__resample_vertical_gather(stbir_info, split_info, y, in_first_scanline, in_last_scanline, vertical_coefficients ); - - ++vertical_contributors; - vertical_coefficients += stbir_info->vertical.coefficient_width; + stbir__resample_vertical_gather(stbir_info, split_info, y, in_first_scanline, in_last_scanline, vertical_coefficients, slice ); } } #define STBIR__FLOAT_EMPTY_MARKER 3.0e+38F #define STBIR__FLOAT_BUFFER_IS_EMPTY(ptr) ((ptr)[0]==STBIR__FLOAT_EMPTY_MARKER) -static void stbir__encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info) +static void stbir__encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info, stbir__column_slice const * slice) { // evict a scanline out into the output buffer float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index ); // dump the scanline out - stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), ring_buffer_entry, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__encode_scanline( stbir_info, ring_buffer_entry, split_info->ring_buffer_first_scanline, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // mark it as empty ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER; @@ -6388,17 +7687,17 @@ static void stbir__encode_first_scanline_from_scatter(stbir__info const * stbir_ split_info->ring_buffer_begin_index = 0; } -static void stbir__horizontal_resample_and_encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info) +static void stbir__horizontal_resample_and_encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info, stbir__column_slice const * slice) { // evict a scanline out into the output buffer float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index ); // Now resample it into the buffer. - stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, ring_buffer_entry STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, ring_buffer_entry, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // dump the scanline out - stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), split_info->vertical_buffer, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__encode_scanline( stbir_info, split_info->vertical_buffer, split_info->ring_buffer_first_scanline, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // mark it as empty ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER; @@ -6439,9 +7738,9 @@ static void stbir__resample_vertical_scatter(stbir__info const * stbir_info, stb STBIR_PROFILE_END( vertical ); } -typedef void stbir__handle_scanline_for_scatter_func(stbir__info const * stbir_info, stbir__per_split_info* split_info); +typedef void stbir__handle_scanline_for_scatter_func(stbir__info const * stbir_info, stbir__per_split_info* split_info, stbir__column_slice const * slice); -static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count ) +static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count, stbir__column_slice const * slice ) { int y, start_output_y, end_output_y, start_input_y, end_input_y; stbir__contributors* vertical_contributors = stbir_info->vertical.contributors; @@ -6450,8 +7749,7 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ void * scanline_scatter_buffer; void * scanline_scatter_buffer_end; int on_first_input_y, last_input_y; - int width = (stbir_info->vertical_first) ? ( stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1 ) : stbir_info->horizontal.scale_info.output_sub_size; - int width_times_channels = stbir_info->effective_channels * width; + ptrdiff_t width_times_channels = stbir__vertical_width_times_channels( stbir_info, slice ); STBIR_ASSERT( !stbir_info->vertical.is_gather ); @@ -6464,20 +7762,19 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ // adjust for starting offset start_input_y y = start_input_y + stbir_info->vertical.filter_pixel_margin; vertical_contributors += y ; - vertical_coefficients += stbir_info->vertical.coefficient_width * y; + vertical_coefficients += (size_t)stbir_info->vertical.coefficient_width * y; if ( stbir_info->vertical_first ) { handle_scanline_for_scatter = stbir__horizontal_resample_and_encode_first_scanline_from_scatter; scanline_scatter_buffer = split_info->decode_buffer; - scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * (stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1); } else { handle_scanline_for_scatter = stbir__encode_first_scanline_from_scatter; scanline_scatter_buffer = split_info->vertical_buffer; - scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * stbir_info->horizontal.scale_info.output_sub_size; } + scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * width_times_channels; // initialize the ring buffer for scattering split_info->ring_buffer_first_scanline = start_output_y; @@ -6494,7 +7791,8 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ } // do the loop in input space - on_first_input_y = 1; last_input_y = start_input_y; + on_first_input_y = 1; + last_input_y = start_input_y; for (y = start_input_y ; y < end_input_y; y++) { int out_first_scanline, out_last_scanline; @@ -6502,11 +7800,10 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ out_first_scanline = vertical_contributors->n0; out_last_scanline = vertical_contributors->n1; - STBIR_ASSERT(out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); - - if ( ( out_last_scanline >= out_first_scanline ) && ( ( ( out_first_scanline >= start_output_y ) && ( out_first_scanline < end_output_y ) ) || ( ( out_last_scanline >= start_output_y ) && ( out_last_scanline < end_output_y ) ) ) ) + // this input row is ours if the output rows it contributes to overlap the split's rows at all + if ( ( out_last_scanline >= out_first_scanline ) && ( out_last_scanline >= start_output_y ) && ( out_first_scanline < end_output_y ) ) { - float const * vc = vertical_coefficients; + float const * coefficients = vertical_coefficients; // keep track of the range actually seen for the next resize last_input_y = y; @@ -6517,13 +7814,15 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ // clip the region if ( out_first_scanline < start_output_y ) { - vc += start_output_y - out_first_scanline; + coefficients += start_output_y - out_first_scanline; out_first_scanline = start_output_y; } if ( out_last_scanline >= end_output_y ) out_last_scanline = end_output_y - 1; + STBIR_ASSERT( out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries ); // (the ring is capped to the band's rows, so after the clip) + // if very first scanline, init the index if (split_info->ring_buffer_begin_index < 0) split_info->ring_buffer_begin_index = out_first_scanline - start_output_y; @@ -6531,21 +7830,21 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ STBIR_ASSERT( split_info->ring_buffer_begin_index <= out_first_scanline ); // Decode the nth scanline from the source image into the decode buffer. - stbir__decode_scanline( stbir_info, y, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__decode_scanline( stbir_info, y, split_info->decode_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // When horizontal first, we resample horizontally into the vertical buffer before we scatter it out if ( !stbir_info->vertical_first ) - stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, split_info->decode_buffer, slice STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); // Now it's sitting in the buffer ready to be distributed into the ring buffers. - // evict from the ringbuffer, if we need are full + // evict from the ring buffer if it is full and this row needs a new entry if ( ( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) == stbir_info->ring_buffer_num_entries ) && ( out_last_scanline > split_info->ring_buffer_last_scanline ) ) - handle_scanline_for_scatter( stbir_info, split_info ); + handle_scanline_for_scatter( stbir_info, split_info, slice ); // Now the horizontal buffer is ready to write to all ring buffer rows, so do it. - stbir__resample_vertical_scatter(stbir_info, split_info, out_first_scanline, out_last_scanline, vc, (float*)scanline_scatter_buffer, (float*)scanline_scatter_buffer_end ); + stbir__resample_vertical_scatter(stbir_info, split_info, out_first_scanline, out_last_scanline, coefficients, (float*)scanline_scatter_buffer, (float*)scanline_scatter_buffer_end ); // update the end of the buffer if ( out_last_scanline > split_info->ring_buffer_last_scanline ) @@ -6557,7 +7856,7 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ // now evict the scanlines that are left over in the ring buffer while ( split_info->ring_buffer_first_scanline < end_output_y ) - handle_scanline_for_scatter(stbir_info, split_info); + handle_scanline_for_scatter(stbir_info, split_info, slice); // update the end_input_y if we do multiple resizes with the same data ++last_input_y; @@ -6567,18 +7866,15 @@ static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir_ } -static stbir__kernel_callback * stbir__builtin_kernels[] = { 0, stbir__filter_trapezoid, stbir__filter_triangle, stbir__filter_cubic, stbir__filter_catmullrom, stbir__filter_mitchell, stbir__filter_point }; -static stbir__support_callback * stbir__builtin_supports[] = { 0, stbir__support_trapezoid, stbir__support_one, stbir__support_two, stbir__support_two, stbir__support_two, stbir__support_zeropoint5 }; - -static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir__kernel_callback * kernel, stbir__support_callback * support, stbir_edge edge, stbir__scale_info * scale_info, int always_gather, void * user_data ) +static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir__kernel_callback_new * kernel_new, stbir__support_callback_new * support_new, stbir__radius_callback_new * radius, stbir__kernel_callback * kernel_old, stbir__support_callback * support_old, stbir_edge edge, stbir__scale_info * scale_info, int always_gather, void * user_data ) { // set filter if (filter == 0) { filter = STBIR_DEFAULT_FILTER_DOWNSAMPLE; // default to downsample - if (scale_info->scale >= ( 1.0f - stbir__small_float ) ) + if ( stbir__scale_is_upsample( scale_info ) ) { - if ( (scale_info->scale <= ( 1.0f + stbir__small_float ) ) && ( STBIR_CEILF(scale_info->pixel_shift) == scale_info->pixel_shift ) ) + if ( stbir__scale_is_copy( scale_info ) ) filter = STBIR_FILTER_POINT_SAMPLE; else filter = STBIR_DEFAULT_FILTER_UPSAMPLE; @@ -6588,24 +7884,32 @@ static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir STBIR_ASSERT(samp->filter_enum != 0); STBIR_ASSERT((unsigned)samp->filter_enum < STBIR_FILTER_OTHER); - samp->filter_kernel = stbir__builtin_kernels[ filter ]; - samp->filter_support = stbir__builtin_supports[ filter ]; + samp->filter_kernel_new = stbir__builtin_kernels[ filter ]; + samp->filter_kernel_old = 0; + samp->filter_support_new = stbir__builtin_supports[ filter ]; + samp->filter_radius = stbir__builtin_radii[ filter ]; + samp->filter_support_old = 0; - if ( kernel && support ) + // a user filter is a kernel with a support or a radius (stbir_set_filter_callbacks_new refuses both), or an old + // float kernel with an old float support (stbir_set_filter_callbacks) + if ( ( kernel_new && ( support_new || radius ) ) || ( kernel_old && support_old ) ) { - samp->filter_kernel = kernel; - samp->filter_support = support; + samp->filter_kernel_new = kernel_new; + samp->filter_kernel_old = kernel_old; + samp->filter_support_new = support_new; + samp->filter_radius = radius; + samp->filter_support_old = support_old; samp->filter_enum = STBIR_FILTER_OTHER; } samp->edge = edge; - samp->filter_pixel_width = stbir__get_filter_pixel_width (samp->filter_support, scale_info->scale, user_data ); + samp->filter_pixel_width = stbir__get_filter_pixel_width( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, scale_info, user_data ); // Gather is always better, but in extreme downsamples, you have to most or all of the data in memory // For horizontal, we always have all the pixels, so we always use gather here (always_gather==1). // For vertical, we use gather if scaling up (which means we will have samp->filter_pixel_width // scanlines in memory at once). samp->is_gather = 0; - if ( scale_info->scale >= ( 1.0f - stbir__small_float ) ) + if ( stbir__scale_is_upsample( scale_info ) ) samp->is_gather = 1; else if ( ( always_gather ) || ( samp->filter_pixel_width <= STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT ) ) samp->is_gather = 2; @@ -6621,7 +7925,7 @@ static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir // overhang, and once for the right side). We only do this for edge mode, since the other // modes can just re-edge clamp back in again. if ( edge == STBIR_EDGE_WRAP ) - if ( samp->filter_pixel_width > ( scale_info->input_full_size * 3 ) ) + if ( ( samp->filter_pixel_width / 3 ) > scale_info->input_full_size ) samp->filter_pixel_width = scale_info->input_full_size * 3; // This is how much to expand buffers to account for filters seeking outside @@ -6638,104 +7942,103 @@ static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir samp->num_contributors = stbir__get_contributors(samp, samp->is_gather); - samp->contributors_size = samp->num_contributors * sizeof(stbir__contributors); - samp->coefficients_size = samp->num_contributors * samp->coefficient_width * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra sizeof(float) is padding + samp->entry_floats = 0; + samp->shared_period = 0; + samp->shared_prologue = 0; + samp->shared_epilogue = 0; + samp->contributors_size = (size_t)samp->num_contributors * sizeof(stbir__contributors); + samp->coefficients_size = (size_t)samp->num_contributors * STBIR__COLUMN_ENTRY_FLOATS( samp->coefficient_width ) * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // room for the horizontal's column entry ints, plus padding +} - samp->gather_prescatter_contributors = 0; - samp->gather_prescatter_coefficients = 0; - if ( samp->is_gather == 0 ) +// Walks input pixels from start toward end (step is -1 or 1) and returns the last one that contributes to any output, +// or fallback if none did. When the output region is only a pixel or two wide the starting guess can be past the +// whole region, so a non contributing pixel only stops the walk once contributing ones have been seen. +static int stbir__last_contributing_input( int start, int end, int step, int fallback, stbir_real in_pixels_radius, stbir_real support, stbir__scale_info const * scale_info ) +{ + int n = start; + int last = fallback; + int seen = 0; + + while ( ( step < 0 ) ? ( n >= end ) : ( n <= end ) ) { - samp->gather_prescatter_coefficient_width = samp->filter_pixel_width; - samp->gather_prescatter_num_contributors = stbir__get_contributors(samp, 2); - samp->gather_prescatter_contributors_size = samp->gather_prescatter_num_contributors * sizeof(stbir__contributors); - samp->gather_prescatter_coefficients_size = samp->gather_prescatter_num_contributors * samp->gather_prescatter_coefficient_width * sizeof(float); + int out_first_pixel, out_last_pixel; + stbir__input_out_pixel_range( scale_info, n, in_pixels_radius, support, &out_first_pixel, &out_last_pixel ); + if ( out_first_pixel > out_last_pixel ) + { + if ( seen ) + break; + } + else + { + last = n; + seen = 1; + } + n += step; } + return last; } static void stbir__get_conservative_extents( stbir__sampler * samp, stbir__contributors * range, void * user_data ) { - float scale = samp->scale_info.scale; - float out_shift = samp->scale_info.pixel_shift; - stbir__support_callback * support = samp->filter_support; + stbir_real scale = samp->scale_info.scale; + stbir_real in_pixels_radius = stbir__filter_radius_in_input_pixels( samp->filter_radius, samp->filter_support_new, samp->filter_support_old, &samp->scale_info, user_data ); int input_full_size = samp->scale_info.input_full_size; stbir_edge edge = samp->edge; - float inv_scale = samp->scale_info.inv_scale; + stbir_real inv_scale = samp->scale_info.inv_scale; STBIR_ASSERT( samp->is_gather != 0 ); if ( samp->is_gather == 1 ) { int in_first_pixel, in_last_pixel; - float out_filter_radius = support(inv_scale, user_data) * scale; + stbir_real filter_support = in_pixels_radius; + stbir_real out_filter_radius = filter_support * scale; - stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0.5, out_filter_radius, inv_scale, out_shift, input_full_size, edge ); + stbir__output_in_pixel_range( &samp->scale_info, 0, out_filter_radius, filter_support, edge, &in_first_pixel, &in_last_pixel ); range->n0 = in_first_pixel; - stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, ( (float)(samp->scale_info.output_sub_size-1) ) + 0.5f, out_filter_radius, inv_scale, out_shift, input_full_size, edge ); + stbir__output_in_pixel_range( &samp->scale_info, samp->scale_info.output_sub_size - 1, out_filter_radius, filter_support, edge, &in_first_pixel, &in_last_pixel ); range->n1 = in_last_pixel; } else if ( samp->is_gather == 2 ) // downsample gather, refine { - float in_pixels_radius = support(scale, user_data) * inv_scale; + stbir_real filter_support = in_pixels_radius / inv_scale; // in output pixels int filter_pixel_margin = samp->filter_pixel_margin; int output_sub_size = samp->scale_info.output_sub_size; - int input_end; - int n; int in_first_pixel, in_last_pixel; // get a conservative area of the input range - stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0, 0, inv_scale, out_shift, input_full_size, edge ); + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, (stbir_real)0.0, (stbir_real)0.0, samp->scale_info.inv_scale, samp->scale_info.pixel_shift, input_full_size, edge ); range->n0 = in_first_pixel; - stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, (float)output_sub_size, 0, inv_scale, out_shift, input_full_size, edge ); + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, (stbir_real)output_sub_size, (stbir_real)0.0, samp->scale_info.inv_scale, samp->scale_info.pixel_shift, input_full_size, edge ); range->n1 = in_last_pixel; - // now go through the margin to the start of area to find bottom - n = range->n0 + 1; - input_end = -filter_pixel_margin; - while( n >= input_end ) - { - int out_first_pixel, out_last_pixel; - stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size ); - if ( out_first_pixel > out_last_pixel ) - break; - - if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) ) - range->n0 = n; - --n; - } - - // now go through the end of the area through the margin to find top - n = range->n1 - 1; - input_end = n + 1 + filter_pixel_margin; - while( n <= input_end ) - { - int out_first_pixel, out_last_pixel; - stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size ); - if ( out_first_pixel > out_last_pixel ) - break; - if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) ) - range->n1 = n; - ++n; - } + // walk down through the margin from the start of the area to find the bottom, and up from its end to find the top + // (the coefficient generator never goes past input_full_size + filter_pixel_margin - 1) + range->n0 = stbir__last_contributing_input( range->n0 + 1, -filter_pixel_margin, -1, range->n0, in_pixels_radius, filter_support, &samp->scale_info ); + range->n1 = stbir__last_contributing_input( range->n1 - 1, input_full_size + filter_pixel_margin - 1, 1, range->n1, in_pixels_radius, filter_support, &samp->scale_info ); } - if ( samp->edge == STBIR_EDGE_WRAP ) + // when polyphase, the coefficient generator only calculates the first numerator outputs and then copies them + // forward shifted by denominator (exact integer math). the top of the range above was calculated directly at the + // far end of the image in floating point, which can round the other way, so widen by one - and when the fraction is + // an approximation of the scale, the copies drift from the real positions by up to approximated_drift input pixels. + if ( ( samp->scale_info.scale_is_rational ) && ( (int)samp->scale_info.scale_numerator < samp->scale_info.output_sub_size ) ) + range->n1 += 1 + samp->scale_info.approximated_drift; + + // with WRAP the range is allowed to hang off either edge (those pixels get decoded from the other end of + // the scanline), the other modes fold everything back into the scanline so clamp them to it + if ( samp->edge != STBIR_EDGE_WRAP ) { - // if we are wrapping, and we are very close to the image size (so the edges might merge), just use the scanline up to the edge - if ( ( range->n0 > 0 ) && ( range->n1 >= input_full_size ) ) + // for reflect, the margin taps get folded back into the image, and on tiny inputs we + // land beyond, so widen to include them + if ( samp->edge == STBIR_EDGE_REFLECT ) { - int marg = range->n1 - input_full_size + 1; - if ( ( marg + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= range->n0 ) - range->n0 = 0; + if ( range->n0 < 0 ) + range->n1 = stbir__max( range->n1, stbir__min( -range->n0, input_full_size - 1 ) ); + if ( range->n1 >= input_full_size ) + range->n0 = stbir__min( range->n0, stbir__max( 0, ( input_full_size - 1 ) - ( range->n1 - input_full_size ) ) ); // ( 2 x size - n1 - 1, written so it can't overflow at 2^30 ) } - if ( ( range->n0 < 0 ) && ( range->n1 < (input_full_size-1) ) ) - { - int marg = -range->n0; - if ( ( input_full_size - marg - STBIR__MERGE_RUNS_PIXEL_THRESHOLD - 1 ) <= range->n1 ) - range->n1 = input_full_size - 1; - } - } - else - { + // for non-edge-wrap modes, we never read over the edge, so clamp if ( range->n0 < 0 ) range->n0 = 0; @@ -6744,69 +8047,36 @@ static void stbir__get_conservative_extents( stbir__sampler * samp, stbir__contr } } -static void stbir__get_split_info( stbir__per_split_info* split_info, int splits, int output_height, int vertical_pixel_margin, int input_full_height, int is_gather, stbir__contributors * contribs ) +static void stbir__get_split_info( stbir__per_split_info* split_info, int splits, int output_height, int vertical_pixel_margin, int input_full_height, int is_gather, stbir__sampler const * vertical ) { - int i, cur; - int left = output_height; + int start_y = 0; + int remaining = output_height; + int i; - cur = 0; - for( i = 0 ; i < splits ; i++ ) + for (i = 0; i < splits; i++) { - int each; + int rows = remaining / ( splits - i ); - split_info[i].start_output_y = cur; - each = left / ( splits - i ); - split_info[i].end_output_y = cur + each; + split_info[i].start_output_y = start_y; + split_info[i].end_output_y = start_y + rows; - // ok, when we are gathering, we need to make sure we are starting on a y offset that doesn't have - // a "special" set of coefficients. Basically, with exactly the right filter at exactly the right - // resize at exactly the right phase, some of the coefficents can be zero. When they are zero, we - // don't process them at all. But this leads to a tricky thing with the thread splits, where we - // might have a set of two coeffs like this for example: (4,4) and (3,6). The 4,4 means there was - // just one single coeff because things worked out perfectly (normally, they all have 4 coeffs - // like the range 3,6. The problem is that if we start right on the (4,4) on a brand new thread, - // then when we get to (3,6), we don't have the "3" sample in memory (because we didn't load - // it on the initial (4,4) range because it didn't have a 3 (we only add new samples that are - // larger than our existing samples - it's just how the eviction works). So, our solution here - // is pretty simple, if we start right on a range that has samples that start earlier, then we - // simply bump up our previous thread split range to include it, and then start this threads - // range with the smaller sample. It just moves one scanline from one thread split to another, - // so that we end with the unusual one, instead of start with it. To do this, we check 2-4 - // sample at each thread split start and then occassionally move them. - - if ( ( is_gather ) && ( i ) ) + // Don't start a split on a row whose range starts later than a following row's (see stbir__lowest_scanline_needed): + // move the rows up to that one into the previous split, so it ends with the unusual row instead of this one + // starting on it. Only this split's rows are looked at (the next split's first row is its own problem). + if ( ( is_gather ) && ( i ) && ( rows > 1 ) ) { - stbir__contributors * small_contribs; - int j, smallest, stop, start_n0; - stbir__contributors * split_contribs = contribs + cur; + int look = vertical_pixel_margin * 3; + int move; - // scan for a max of 3x the filter width or until the next thread split - stop = vertical_pixel_margin * 3; - if ( each < stop ) - stop = each; - - // loops a few times before early out - smallest = 0; - small_contribs = split_contribs; - start_n0 = small_contribs->n0; - for( j = 1 ; j <= stop ; j++ ) - { - ++split_contribs; - if ( split_contribs->n0 > start_n0 ) - break; - if ( split_contribs->n0 < small_contribs->n0 ) - { - small_contribs = split_contribs; - smallest = j; - } - } - - split_info[i-1].end_output_y += smallest; - split_info[i].start_output_y += smallest; + if ( look > ( rows - 1 ) ) + look = ( rows - 1 ); + stbir__lowest_scanline_needed( vertical, start_y, look, &move ); + split_info[i-1].end_output_y += move; + split_info[i].start_output_y += move; } - cur += each; - left -= each; + start_y += rows; + remaining -= rows; // scatter range (updated to minimum as you run it) split_info[i].start_input_y = -vertical_pixel_margin; @@ -6823,34 +8093,47 @@ static void stbir__free_internal_mem( stbir__info *info ) #ifndef STBIR__SEPARATE_ALLOCATIONS STBIR__FREE_AND_CLEAR( info->alloced_mem ); #else + // a build whose allocation failed part way comes through here too: info and its pointer arrays are zeroed when + // allocated, so every pointer is either an allocation or null int i,j; - if ( ( info->vertical.gather_prescatter_contributors ) && ( (void*)info->vertical.gather_prescatter_contributors != (void*)info->split_info[0].decode_buffer ) ) + if ( info->split_info ) { - STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_coefficients ); - STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_contributors ); - } - for( i = 0 ; i < info->splits ; i++ ) - { - for( j = 0 ; j < info->alloc_ring_buffer_num_entries ; j++ ) + for( i = 0 ; i < info->splits ; i++ ) { - #ifdef STBIR_SIMD8 - if ( info->effective_channels == 3 ) - --info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer - #endif - STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers[j] ); - } + stbir__per_split_info * split = info->split_info + i; - #ifdef STBIR_SIMD8 - if ( info->effective_channels == 3 ) - --info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer - #endif - STBIR__FREE_AND_CLEAR( info->split_info[i].decode_buffer ); - STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers ); - STBIR__FREE_AND_CLEAR( info->split_info[i].vertical_buffer ); + if ( split->ring_buffers ) + { + for( j = 0 ; j < info->alloc_ring_buffer_num_entries ; j++ ) + { + float * ring_buffer = split->ring_buffers[j]; + if ( ring_buffer ) + { + #ifdef STBIR_SIMD8 + if ( info->effective_channels == 3 ) + --ring_buffer; // avx in 3 channel mode needs one float at the start of the buffer + #endif + STBIR_FREE( ring_buffer, info->user_data ); + } + } + STBIR__FREE_AND_CLEAR( split->ring_buffers ); + } + + if ( split->decode_buffer_alloc ) + { + #ifdef STBIR_SIMD8 + if ( info->effective_channels == 3 ) + split->decode_buffer_alloc = ( (float*)split->decode_buffer_alloc ) - 1; // avx in 3 channel mode needs one float at the start of the buffer + #endif + STBIR__FREE_AND_CLEAR( split->decode_buffer_alloc ); + } + STBIR__FREE_AND_CLEAR( split->vertical_buffer_alloc ); + } + STBIR__FREE_AND_CLEAR( info->split_info ); } - STBIR__FREE_AND_CLEAR( info->split_info ); - if ( info->vertical.coefficients != info->horizontal.coefficients ) + STBIR__FREE_AND_CLEAR( info->column_slices ); + if ( info->vertical.contributors != info->horizontal.contributors ) // (when shared, the vertical's coefficients point two ints into the horizontal's) { STBIR__FREE_AND_CLEAR( info->vertical.coefficients ); STBIR__FREE_AND_CLEAR( info->vertical.contributors ); @@ -6890,142 +8173,114 @@ static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_channe 0, stbir__horizontal_gather_1_channels_funcs, stbir__horizontal_gather_2_channels_funcs, stbir__horizontal_gather_3_channels_funcs, stbir__horizontal_gather_4_channels_funcs, 0,0, stbir__horizontal_gather_7_channels_funcs }; -// there are six resize classifications: 0 == vertical scatter, 1 == vertical gather < 1x scale, 2 == vertical gather 1x-2x scale, 4 == vertical gather < 3x scale, 4 == vertical gather > 3x scale, 5 == <=4 pixel height, 6 == <=4 pixel wide column -#define STBIR_RESIZE_CLASSIFICATIONS 8 - -static float stbir__compute_weights[5][STBIR_RESIZE_CLASSIFICATIONS][4]= // 5 = 0=1chan, 1=2chan, 2=3chan, 3=4chan, 4=7chan +// the polyphase period 1 kernels, [channels][coefficient count] (see the horizontal kernel section) +static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_period1_funcs[8] = { - { - { 1.00000f, 1.00000f, 0.31250f, 1.00000f }, - { 0.56250f, 0.59375f, 0.00000f, 0.96875f }, - { 1.00000f, 0.06250f, 0.00000f, 1.00000f }, - { 0.00000f, 0.09375f, 1.00000f, 1.00000f }, - { 1.00000f, 1.00000f, 0.31250f, 1.00000f }, - { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, - { 1.00000f, 1.00000f, 0.06250f, 1.00000f }, - { 0.00000f, 1.00000f, 0.00000f, 0.03125f }, - }, { - { 0.00000f, 0.84375f, 0.00000f, 0.03125f }, - { 0.09375f, 0.93750f, 0.00000f, 0.78125f }, - { 0.87500f, 0.21875f, 0.00000f, 0.96875f }, - { 0.09375f, 0.09375f, 1.00000f, 1.00000f }, - { 0.00000f, 0.84375f, 0.00000f, 0.03125f }, - { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, - { 1.00000f, 1.00000f, 0.06250f, 1.00000f }, - { 0.00000f, 1.00000f, 0.00000f, 0.53125f }, - }, { - { 0.00000f, 0.53125f, 0.00000f, 0.03125f }, - { 0.06250f, 0.96875f, 0.00000f, 0.53125f }, - { 0.87500f, 0.18750f, 0.00000f, 0.93750f }, - { 0.00000f, 0.09375f, 1.00000f, 1.00000f }, - { 0.00000f, 0.53125f, 0.00000f, 0.03125f }, - { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, - { 1.00000f, 1.00000f, 0.06250f, 1.00000f }, - { 0.00000f, 1.00000f, 0.00000f, 0.56250f }, - }, { - { 0.00000f, 0.50000f, 0.00000f, 0.71875f }, - { 0.06250f, 0.84375f, 0.00000f, 0.87500f }, - { 1.00000f, 0.50000f, 0.50000f, 0.96875f }, - { 1.00000f, 0.09375f, 0.31250f, 0.50000f }, - { 0.00000f, 0.50000f, 0.00000f, 0.71875f }, - { 1.00000f, 0.03125f, 0.03125f, 0.53125f }, - { 1.00000f, 1.00000f, 0.06250f, 1.00000f }, - { 0.00000f, 1.00000f, 0.03125f, 0.18750f }, - }, { - { 0.00000f, 0.59375f, 0.00000f, 0.96875f }, - { 0.06250f, 0.81250f, 0.06250f, 0.59375f }, - { 0.75000f, 0.43750f, 0.12500f, 0.96875f }, - { 0.87500f, 0.06250f, 0.18750f, 0.43750f }, - { 0.00000f, 0.59375f, 0.00000f, 0.96875f }, - { 0.15625f, 0.12500f, 1.00000f, 1.00000f }, - { 1.00000f, 1.00000f, 0.06250f, 1.00000f }, - { 0.00000f, 1.00000f, 0.03125f, 0.34375f }, - } + 0, stbir__horizontal_gather_1_channels_period1_funcs, stbir__horizontal_gather_2_channels_period1_funcs, stbir__horizontal_gather_3_channels_period1_funcs, stbir__horizontal_gather_4_channels_period1_funcs, 0,0, stbir__horizontal_gather_7_channels_period1_funcs }; -// structure that allow us to query and override info for training the costs -typedef struct STBIR__V_FIRST_INFO -{ - double v_cost, h_cost; - int control_v_first; // 0 = no control, 1 = force hori, 2 = force vert - int v_first; - int v_resize_classification; - int is_gather; -} STBIR__V_FIRST_INFO; +// debug overrides (the tests set these; they are statics, so per translation unit): 0 = off +static int stbir__force_vertical_first = 0; // 1 = horizontal first, 2 = vertical first (dotimings, refcheck -vfirst) +static int stbir__chose_vertical_first = 0; // (read only) what the cost model picked for the last build +static int stbir__split_mode = 0; // 1 = row bands only, 2 = column groups only (refcheck_threads -row_split / -colsplit) +static int stbir__split_force_cols = 0; // force this many column groups, the bands are splits / this (refcheck_threads -grid) +static int stbir__column_slices = 0; // force this many column slices instead of the L2 fit (refcheck_time cfg N:) +static int stbir__4K_vertical_dest_shift = 0; // bytes added to the vertical gather destination's page offset (refcheck_time -vshift) -#ifdef STBIR__V_FIRST_INFO_BUFFER -static STBIR__V_FIRST_INFO STBIR__V_FIRST_INFO_BUFFER = {0}; -#define STBIR__V_FIRST_INFO_POINTER &STBIR__V_FIRST_INFO_BUFFER -#else -#define STBIR__V_FIRST_INFO_POINTER 0 -#endif - -// Figure out whether to scale along the horizontal or vertical first. -// This only *super* important when you are scaling by a massively -// different amount in the vertical vs the horizontal (for example, if -// you are scaling by 2x in the width, and 0.5x in the height, then you -// want to do the vertical scale first, because it's around 3x faster -// in that order. +// Vertical first or horizontal first? It matters most when the two axes scale very differently (2x wider and half the +// height: the vertical first order is around 3x faster), and 20-40% in ordinary cases. The decision is a kernel cost +// model: both orders move the same decoded pixels, so the question is only how much each kernel charges per byte it +// streams and the multiplies each direction does (vertical is more SIMDable than horizontal). We calculate the costs as: // -// In more normal circumstances, this makes a 20-40% differences, so -// it's good to get right, but not critical. The normal way that you -// decide which direction goes first is just figuring out which -// direction does more multiplies. But with modern CPUs with their -// fancy caches and SIMD and high IPC abilities, so there's just a lot -// more that goes into it. +// vertical first: (taps_v + 1) / s_h * rv + (1/s_h + 1) * rh (vertical over input-width rows, horizontal over output rows) +// horizontal first: (1/s_v) * (1/s_h + 1) * rh + (taps_v + 1) * rv (horizontal over every input row, vertical over output-width rows) +// (1/s = input/output per axis, with WRAP's margin pixels and scanlines added to the input) // -// My handwavy sort of solution is to have an app that does a whole -// bunch of timing for both vertical and horizontal first modes, -// and then another app that can read lots of these timing files -// and try to search for the best weights to use. Dotimings.c -// is the app that does a bunch of timings, and vf_train.c is the -// app that solves for the best weights (and shows how well it -// does currently). +// rh and rv are the scale factors for the horizontal and vertical costs. We then measure a bunch of horizontal +// first and vertical first resizes on a bunch of machines to train small tables to give us a relative speed of +// each direction. -static int stbir__should_do_vertical_first( float weights_table[STBIR_RESIZE_CLASSIFICATIONS][4], int horizontal_filter_pixel_width, float horizontal_scale, int horizontal_output_size, int vertical_filter_pixel_width, float vertical_scale, int vertical_output_size, int is_gather, STBIR__V_FIRST_INFO * info ) +// horizontal tap count -> table column: 1, 2, 3, 4, 5-6, 7-8, 9-12, 13-17, 18+ +static int stbir__horizontal_cost_bucket( int taps ) { - double v_cost, h_cost; - float * weights; - int vertical_first; - int v_classification; + if ( taps <= 4 ) return ( taps < 1 ) ? 0 : taps - 1; + if ( taps <= 6 ) return 4; + if ( taps <= 8 ) return 5; + if ( taps <= 12 ) return 6; + if ( taps <= 17 ) return 7; + return 8; +} - // categorize the resize into buckets - if ( ( vertical_output_size <= 4 ) || ( horizontal_output_size <= 4 ) ) - v_classification = ( vertical_output_size < horizontal_output_size ) ? 6 : 7; - else if ( ( !is_gather ) && ( ( vertical_output_size <= 16 ) || ( horizontal_output_size <= 16 ) ) ) - v_classification = 4; - else if ( vertical_scale <= 1.0f ) - v_classification = ( is_gather ) ? 1 : 0; - else if ( vertical_scale <= 2.0f) - v_classification = 2; - else if ( vertical_scale <= 3.0f) - v_classification = 3; - else - v_classification = 5; // everything bigger than 3x +// [1,2,3,4,7 channels][bucket]: horizontal kernel cost per byte streamed, relative to the vertical kernel's +static float stbir__horizontal_cost[5][9] = +{ + { 12.472f, 6.718f, 6.656f, 8.008f, 5.957f, 6.068f, 6.474f, 7.646f, 10.184f }, + { 8.067f, 4.192f, 5.091f, 4.906f, 4.816f, 5.584f, 5.741f, 6.971f, 6.182f }, + { 5.495f, 3.718f, 3.206f, 7.861f, 4.816f, 5.584f, 5.741f, 5.584f, 6.068f }, + { 3.651f, 2.135f, 2.791f, 3.583f, 2.950f, 3.583f, 4.154f, 4.192f, 4.861f }, + { 2.434f, 2.001f, 2.521f, 3.583f, 2.817f, 3.236f, 3.550f, 3.822f, 4.271f }, +}; - // use the right weights - weights = weights_table[ v_classification ]; +// the same when the horizontal is an integer downsample (the polyphase period 1 kernels) +static float stbir__horizontal_cost_period1[5][9] = +{ + { 12.406f, 6.718f, 6.534f, 5.186f, 5.114f, 5.138f, 6.656f, 5.431f, 10.374f }, + { 8.001f, 4.154f, 3.583f, 4.003f, 3.276f, 4.473f, 3.683f, 4.231f, 6.013f }, + { 5.024f, 3.033f, 3.583f, 3.683f, 3.268f, 4.432f, 3.650f, 4.231f, 5.283f }, + { 2.976f, 2.076f, 2.665f, 2.452f, 2.354f, 2.766f, 2.715f, 3.327f, 4.684f }, + { 2.184f, 1.928f, 2.342f, 1.983f, 2.233f, 3.005f, 2.715f, 3.421f, 3.858f }, +}; - // this is the costs when you don't take into account modern CPUs with high ipc and simd and caches - wish we had a better estimate - h_cost = (float)horizontal_filter_pixel_width * weights[0] + horizontal_scale * (float)vertical_filter_pixel_width * weights[1]; - v_cost = (float)vertical_filter_pixel_width * weights[2] + vertical_scale * (float)horizontal_filter_pixel_width * weights[3]; +// a x b as a signed 128 bit value, for the exact cost comparison below +static void stbir__mul_64x64( stbir_int64 a, stbir_int64 b, stbir_int64 * hi, stbir_uint64 * lo ) +{ + stbir_uint64 ua = (stbir_uint64) a, ub = (stbir_uint64) b; + stbir_uint64 a0 = ua & 0xffffffffu, a1 = ua >> 32, b0 = ub & 0xffffffffu, b1 = ub >> 32; + stbir_uint64 p00 = a0 * b0, p01 = a0 * b1, p10 = a1 * b0, p11 = a1 * b1; + stbir_uint64 mid = ( p00 >> 32 ) + ( p01 & 0xffffffffu ) + ( p10 & 0xffffffffu ); + stbir_uint64 uhi = p11 + ( p01 >> 32 ) + ( p10 >> 32 ) + ( mid >> 32 ); + *lo = ( mid << 32 ) | ( p00 & 0xffffffffu ); + if ( a < 0 ) uhi -= ub; // the unsigned product of the two's complements, corrected to the signed one + if ( b < 0 ) uhi -= ua; + *hi = (stbir_int64) uhi; +} - // use computation estimate to decide vertical first or not - vertical_first = ( v_cost <= h_cost ) ? 1 : 0; +// a x b <= c x d, exactly +static int stbir__products_le( stbir_int64 a, stbir_int64 b, stbir_int64 c, stbir_int64 d ) +{ + stbir_int64 lhi, rhi; + stbir_uint64 llo, rlo; + stbir__mul_64x64( a, b, &lhi, &llo ); + stbir__mul_64x64( c, d, &rhi, &rlo ); + return ( lhi < rhi ) || ( ( lhi == rhi ) && ( llo <= rlo ) ); +} - // save these, if requested - if ( info ) - { - info->h_cost = h_cost; - info->v_cost = v_cost; - info->v_resize_classification = v_classification; - info->v_first = vertical_first; - info->is_gather = is_gather; - } +// WRAP is the one edge mode that changes the amount of work: the decoded rows carry the filter's margin pixels on both +// ends and the vertical runs over margin scanlines, so the input is effectively taps wider / taller. +static int stbir__should_do_vertical_first( int channel_index, int horizontal_filter_pixel_width, int horizontal_input_size, int horizontal_output_size, stbir_edge horizontal_edge, int horizontal_is_period1, + int vertical_filter_pixel_width, int vertical_input_size, int vertical_output_size, stbir_edge vertical_edge ) +{ + stbir_int64 horizontal_input, vertical_input, horizontal_output, vertical_output, vertical_taps, cost_ratio; + int bucket, vertical_first; - // and this allows us to override everything for testing (see dotiming.c) - if ( ( info ) && ( info->control_v_first ) ) - vertical_first = ( info->control_v_first == 2 ) ? 1 : 0; + // the horizontal kernel's cost relative to the vertical's, in 1/1000ths + bucket = stbir__horizontal_cost_bucket( horizontal_filter_pixel_width ); + cost_ratio = (stbir_int64) ( ( horizontal_is_period1 ? stbir__horizontal_cost_period1 : stbir__horizontal_cost )[ channel_index ][ bucket ] * 1000.0f + 0.5f ); + + // the inputs with their wrap margins + horizontal_input = (stbir_int64) horizontal_input_size + ( ( horizontal_edge == STBIR_EDGE_WRAP ) ? horizontal_filter_pixel_width : 0 ); + vertical_input = (stbir_int64) vertical_input_size + ( ( vertical_edge == STBIR_EDGE_WRAP ) ? vertical_filter_pixel_width : 0 ); + horizontal_output = horizontal_output_size; + vertical_output = vertical_output_size; + vertical_taps = (stbir_int64) vertical_filter_pixel_width + 1; + + vertical_first = stbir__products_le( vertical_taps * vertical_output, ( horizontal_input - horizontal_output ) * 1000, + cost_ratio * ( horizontal_input + horizontal_output ), vertical_input - vertical_output ); + + // the test hook: report the choice, and let the tests force it + stbir__chose_vertical_first = vertical_first; + if ( stbir__force_vertical_first ) + vertical_first = ( stbir__force_vertical_first == 2 ) ? 1 : 0; return vertical_first; } @@ -7045,7 +8300,641 @@ static stbir_internal_pixel_layout stbir__pixel_layout_convert_public_to_interna STBIRI_RGBA_PM, STBIRI_BGRA_PM, STBIRI_ARGB_PM, STBIRI_ABGR_PM, STBIRI_RA_PM, STBIRI_AR_PM, }; -static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sampler * horizontal, stbir__sampler * vertical, stbir__contributors * conservative, stbir_pixel_layout input_pixel_layout_public, stbir_pixel_layout output_pixel_layout_public, int splits, int new_x, int new_y, int fast_alpha, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) +// We need a good guess at the L2 size for the column sizing (CACHE SIZE in the docs). We get this size, in order of preference as: +// - stbir_l2_cache_size_override, if you set it to something non-zero before resizing (static, starts at 0, bytes) +// - the OS, if you define one of these before including this header (we don't include the headers for you): +// STBIR_ASK_WINDOWS_FOR_L2_SIZE - needs and (GetLogicalProcessorInformationEx into an _alloca buffer) +// STBIR_ASK_LINUX_FOR_L2_SIZE - needs (sysconf _SC_LEVEL2_CACHE_SIZE) +// STBIR_ASK_MACOS_FOR_L2_SIZE - needs (sysctlbyname hw.l2cachesize) +// - cpuid on x86/x64 (leaf 0x80000006, the L2 size in KB - Intel and AMD both fill it in) +// - 256 KB if none of that applies or answers +// An L2 of 1 MB or more is clamped to 512 KB to keep total resize internal memory use down. + +static int stbir_l2_cache_size_override = 0; // bytes, 0 = detect + +#if ( defined( _M_X64 ) || defined( _M_IX86 ) || defined( __x86_64__ ) || defined( __i386__ ) ) && !defined( _M_ARM64EC ) + #define STBIR__L2_FROM_CPUID + #ifdef _MSC_VER + #include + #else + #include + #endif +#endif + +// (asked on every build - cpuid is a few hundred cycles, and no cache means nothing shared between threads building at once) +static int stbir__l2_cache_size( void ) +{ + int bytes = 0; + + if ( stbir_l2_cache_size_override > 0 ) + return stbir_l2_cache_size_override; + + #if defined( STBIR_ASK_WINDOWS_FOR_L2_SIZE ) + { + // the level 2 data or unified cache of the first processor group that reports one. the list is a few hundred + // bytes per cache level per core, so it goes on the stack (a 64 core machine's is ~30 KB; past 256 KB fall through) + DWORD len = 0; + GetLogicalProcessorInformationEx( RelationCache, 0, &len ); + if ( ( len ) && ( len <= 256 * 1024 ) ) + { + char * buf = (char*) _alloca( len ); + if ( GetLogicalProcessorInformationEx( RelationCache, (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) buf, &len ) ) + { + DWORD ofs = 0; + while ( ofs < len ) + { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX * info = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) ( buf + ofs ); + if ( ( info->Relationship == RelationCache ) && ( info->Cache.Level == 2 ) && ( ( info->Cache.Type == CacheData ) || ( info->Cache.Type == CacheUnified ) ) ) + { + bytes = (int) info->Cache.CacheSize; + break; + } + ofs += info->Size; + } + } + } + } + #elif defined( STBIR_ASK_LINUX_FOR_L2_SIZE ) + { + long v = sysconf( _SC_LEVEL2_CACHE_SIZE ); + if ( v > 0 ) + bytes = (int) v; + } + #elif defined( STBIR_ASK_MACOS_FOR_L2_SIZE ) + { + stbir_uint64 v = 0; + size_t vlen = sizeof( v ); + if ( ( sysctlbyname( "hw.l2cachesize", &v, &vlen, 0, 0 ) == 0 ) && ( v > 0 ) ) + bytes = (int) v; + } + #endif + + #if defined( STBIR__L2_FROM_CPUID ) + if ( bytes <= 0 ) // no OS query, or it didn't answer + { + unsigned int regs[4] = { 0, 0, 0, 0 }; + #ifdef _MSC_VER + __cpuid( (int*) regs, 0x80000000 ); + if ( regs[0] >= 0x80000006 ) + { + __cpuid( (int*) regs, 0x80000006 ); + bytes = (int) ( regs[2] >> 16 ) * 1024; // ECX[31:16] = L2 size in KB + } + #else + if ( __get_cpuid_max( 0x80000000, 0 ) >= 0x80000006 ) + { + __get_cpuid( 0x80000006, ®s[0], ®s[1], ®s[2], ®s[3] ); + bytes = (int) ( regs[2] >> 16 ) * 1024; + } + #endif + } + #endif + + if ( bytes <= 0 ) + bytes = 256 * 1024; + else if ( bytes >= 1024 * 1024 ) + bytes = 512 * 1024; + + return bytes; +} + +// The splits (what a caller threads over) are a grid of split_cols column groups x split_rows row bands. Any grid gives +// the same pixels and much the same memory (a split holds one slice's buffers either way), so stbir__choose_split_grid +// picks the fastest by estimate, for every count from the request down to one split: +// - a row band re-decodes the vertical filter's taps rows before its first output row, so short bands cost more. +// - column groups are free while they are the slices a single thread would run anyway (the L2 fit); each extra slice +// costs stbir_split_slice_row_cost columns of work per row. A group streams at least stbir_split_min_group_bytes +// of each input row; below stbir_split_group_full_bytes it pays stbir_split_group_cost_pct per (full / bytes - 1) +// per 8 threads, and narrow rows cost (threads / 8)^2 x (64 KB / input row bytes) / stbir_split_stream_cost_div. +// - a thread costs stbir_thread_start_cost of work to start and join, and the count is capped so that the work +// (output pixels x taps x channels / 4) is at least stbir_split_min_work x splits^2. +// - each count is scored by its largest split, and the last threads are dropped while they add less than +// stbir_split_min_thread_gain_pct of a thread's speed-up each over the best smaller count. +// So the build may return fewer splits than asked for (which was always allowed); stbir_split_extra_threads adds some back. + +// the split estimate's tuning (statics so an app can override them before resizing, like stbir_l2_cache_size_override; +// work units are output pixels x taps x channels / 4, roughly nanoseconds of resizing on a fast core) +static int stbir_split_min_group_bytes = 1024; // the least input row a column group streams (bytes) +static int stbir_split_group_full_bytes = 4096; // below this much input row a column group pays the narrow group cost: +static int stbir_split_group_cost_pct = 30; // this percent x ( full / bytes - 1 ) x ( threads / 8 ) of the work +static int stbir_split_stream_cost_div = 170; // narrow rows under many threads: ( threads / 8 )^2 x ( 64 KB / input row bytes ) / this, whatever the grid +static int stbir_split_slice_row_cost = 250; // input columns' worth of work per row for each slice beyond the L2 fit +static int stbir_thread_start_cost = 12000; // starting and joining one thread, in work units (a spinning pool ~1000, a thread per job 50000+) +static int stbir_split_min_thread_gain_pct = 30; // the last threads must each add this percent of a thread's worth of speed-up, or they are dropped +static int stbir_split_min_work = 20000; // the work must be at least this x splits^2 +static int stbir_split_extra_threads = 0; // use this many more splits than the estimate settles on (never more than asked for) +#define STBIR__SPLIT_MAX_SPLITS 256 // the grid search's table + + +// slice boundaries are multiples of this many pixels (so the SIMD loops only run their remnants at the end of the row) +#define STBIR__SLICE_STEP 16 + +// the most slices a row can have (each at least a step wide) +static int stbir__slice_max_count( int output_columns ) +{ + int n = output_columns / STBIR__SLICE_STEP; + return ( n < 1 ) ? 1 : n; +} + + +// The split grid for a request of *splits: cols x rows, and *splits becomes cols * rows (never more than asked for). +static void stbir__choose_split_grid( int * splits, int * out_cols, int * out_rows, int mode, int output_w, int output_h, int input_w, int input_pixel_bytes, int taps_v, float inv_scale_v, int l2_slices, int effective_channels ) +{ + int max_splits = *splits; + int requested_splits; + int max_cols = stbir__slice_max_count( output_w ); // a column group needs at least one slice + int best_cols = 1; + int best_rows = 1; + + if ( max_splits < 1 ) + max_splits = 1; + if ( max_splits > STBIR__SPLIT_MAX_SPLITS ) + max_splits = STBIR__SPLIT_MAX_SPLITS; + requested_splits = max_splits; + + if ( stbir__split_force_cols > 0 ) + { + // Debug: a forced grid. + best_cols = ( stbir__split_force_cols < max_cols ) ? stbir__split_force_cols : max_cols; + if ( best_cols > max_splits ) + best_cols = max_splits; + best_rows = max_splits / best_cols; + } + else if ( mode == 1 ) + { + // Row bands only. + best_rows = ( max_splits < output_h ) ? max_splits : output_h; + } + else if ( mode == 2 ) + { + // Column groups only. + best_cols = ( max_splits < max_cols ) ? max_splits : max_cols; + } + else + { + // Everything in fixed point: times are in 1/65536ths of the single threaded work. + stbir_uint64 work = (stbir_uint64)output_w * (stbir_uint64)output_h * (stbir_uint64)taps_v * (stbir_uint64)effective_channels / 4; + stbir_uint64 band_warmup = (stbir_uint64)( (float)taps_v * 65536.0f / ( ( inv_scale_v > 1.0f ) ? inv_scale_v : 1.0f ) ); // what a band re-decodes before its first row, in output rows' worth of work + stbir_uint64 row_bytes = (stbir_uint64)input_w * (stbir_uint64)input_pixel_bytes; + stbir_uint64 estimate_of[ STBIR__SPLIT_MAX_SPLITS + 1 ]; + int cols_of[ STBIR__SPLIT_MAX_SPLITS + 1 ]; + int rows_of[ STBIR__SPLIT_MAX_SPLITS + 1 ]; + int count, cols; + + if ( work < 1 ) + work = 1; + if ( row_bytes < 1 ) + row_bytes = 1; + + // The thread start/join cost grows with the count: the work must be at least min_work x splits^2. + while ( ( max_splits > 1 ) && ( work < (stbir_uint64)stbir_split_min_work * (stbir_uint64)max_splits * (stbir_uint64)max_splits ) ) + --max_splits; + + // The best grid for every count up to the request, by estimated time (the largest split's - it sets the pace). + for (count = 1; count <= requested_splits; count++) + { + estimate_of[count] = ~(stbir_uint64)0; + cols_of[count] = 1; + rows_of[count] = 1; + + for (cols = 1; cols <= count; cols++) + { + int rows, rows_per_band, extra_slices; + stbir_uint64 group_bytes, narrow_cost, stream_cost, per_row_cost, estimate; + + if ( count % cols ) + continue; + if ( cols > max_cols ) + break; + + // A column group streams its share of each input row, and never less than min_group_bytes of it. + group_bytes = row_bytes / (stbir_uint64)cols; + if ( group_bytes < 1 ) + group_bytes = 1; + if ( ( cols > 1 ) && ( group_bytes < (stbir_uint64)stbir_split_min_group_bytes ) ) + break; + + // Narrow groups cost group_cost_pct per ( full / bytes - 1 ) per 8 threads. + narrow_cost = 0; + if ( ( cols > 1 ) && ( group_bytes < (stbir_uint64)stbir_split_group_full_bytes ) ) + narrow_cost = ( ( (stbir_uint64)stbir_split_group_full_bytes * 65536 ) / group_bytes - 65536 ) * (stbir_uint64)stbir_split_group_cost_pct * (stbir_uint64)count / ( 100 * 8 ); + + // Narrow rows under many threads cost the same whatever the grid (every row is a fresh page, latency bound, and + // the latency climbs with the thread count): the larger of this and the narrow group cost. + stream_cost = ( (stbir_uint64)count * (stbir_uint64)count * 65536 * 65536 ) / ( (stbir_uint64)64 * (stbir_uint64)stbir_split_stream_cost_div * row_bytes ); + if ( stream_cost > narrow_cost ) + narrow_cost = stream_cost; + + rows = count / cols; + if ( rows > output_h ) + continue; + + // The slowest split sets the time: the largest band (rows rounded up) doing the largest group's share of the + // slices (the slice count rounded up to a multiple of the groups), against an even share of the work. + rows_per_band = ( output_h + rows - 1 ) / rows; + extra_slices = ( ( l2_slices + cols - 1 ) / cols ) * cols - l2_slices; + per_row_cost = 65536 + band_warmup / (stbir_uint64)rows_per_band + ( (stbir_uint64)extra_slices * (stbir_uint64)stbir_split_slice_row_cost * 65536 ) / (stbir_uint64)input_w + narrow_cost; + estimate = ( (stbir_uint64)rows_per_band * (stbir_uint64)rows ) * per_row_cost / ( (stbir_uint64)output_h * (stbir_uint64)count ); + estimate += ( (stbir_uint64)count * (stbir_uint64)stbir_thread_start_cost * 65536 ) / work; // plus the threads' own start/join cost + + if ( estimate < estimate_of[count] ) + { + estimate_of[count] = estimate; + cols_of[count] = cols; + rows_of[count] = rows; + } + } + } + + // From the request down, drop threads while they add less than min_thread_gain_pct of a thread's speed-up each + // over the best smaller count (not the next one down: a prime count only has bad grids, which makes the count + // above it look good). + count = max_splits; + while ( count > 1 ) + { + stbir_uint64 speedup = ( estimate_of[1] * 256 ) / estimate_of[count]; + stbir_uint64 best_speedup = 0; + int fewer, best_fewer = 1; + + for (fewer = 1; fewer < count; fewer++) + { + stbir_uint64 speedup_fewer = ( estimate_of[1] * 256 ) / estimate_of[fewer]; + if ( speedup_fewer > best_speedup ) + { + best_speedup = speedup_fewer; + best_fewer = fewer; + } + } + + if ( ( speedup > best_speedup ) && ( ( speedup - best_speedup ) * 100 >= (stbir_uint64)stbir_split_min_thread_gain_pct * 256 * (stbir_uint64)( count - best_fewer ) ) ) + break; + + // The extra threads over the best smaller count weren't worth it: fall back to it and check again. + count = best_fewer; + } + + // The app wants more than the estimate settled on. + if ( stbir_split_extra_threads > 0 ) + { + count += stbir_split_extra_threads; + if ( count > requested_splits ) + count = requested_splits; + // A count no grid fits (more bands than rows): the next one down. + while ( ( count > 1 ) && ( estimate_of[count] == ~(stbir_uint64)0 ) ) + --count; + } + + best_cols = cols_of[count]; + best_rows = rows_of[count]; + } + + if ( best_cols > max_cols ) + best_cols = max_cols; + if ( best_rows > output_h ) + best_rows = output_h; + if ( best_cols < 1 ) + best_cols = 1; + if ( best_rows < 1 ) + best_rows = 1; + + *out_cols = best_cols; + *out_rows = best_rows; + *splits = best_cols * best_rows; +} + +// where slice number slice of num_slices ends +static int stbir__slice_end( int output_columns, int num_slices, int slice ) +{ + stbir_uint64 end; + + if ( slice >= num_slices - 1 ) + return output_columns; + + end = ( (stbir_uint64)( slice + 1 ) * (stbir_uint64)output_columns ) / (stbir_uint64)num_slices; + return ( (int)end / STBIR__SLICE_STEP ) * STBIR__SLICE_STEP; +} + +// the widest slice there can be for a count (for sizing the per slice buffers): the average plus a step of rounding +static int stbir__slice_columns_for( int output_columns, int slices ) +{ + int columns = ( output_columns + slices - 1 ) / slices + STBIR__SLICE_STEP; + if ( columns >= output_columns ) + columns = output_columns; + return columns; +} + +// the input pixels a slice decodes for that many columns (see the sizing in stbir__alloc_internal_mem_and_build_samplers) +static int stbir__slice_in_span_for( stbir__sampler const * horizontal, int slice_columns, int conservative_in_span ) +{ + int span = (int)( slice_columns * horizontal->scale_info.inv_scale ) + 1 + horizontal->filter_pixel_width + 2 * horizontal->coefficient_width + 4; + if ( ( slice_columns >= horizontal->scale_info.output_sub_size ) || ( span > conservative_in_span ) ) + span = conservative_in_span; + return span; +} + +// the ring buffer for one slice: its entries are horizontal output rows (slice_columns wide), or decoded input rows +// (slice_in_span wide) when vertical first +static size_t stbir__ring_bytes_for( stbir__sampler const * horizontal, int slice_columns, int conservative_in_span, int effective_channels, int vertical_first, int ring_entries ) +{ + int row_pixels = vertical_first ? stbir__slice_in_span_for( horizontal, slice_columns, conservative_in_span ) : slice_columns; + return (size_t)ring_entries * ( (size_t)row_pixels * effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING ); +} + +// pick the slice count: the fewest slices whose ring buffer fits the L2 budget (allowing an eighth over - a little +// spill costs less than another slice's re-decode), or the most slices possible (16 columns each) if none fit +static int stbir__choose_column_slices( stbir__sampler const * horizontal, int conservative_in_span, int effective_channels, int vertical_first, int ring_entries ) +{ + int output_columns = horizontal->scale_info.output_sub_size; + size_t budget = (size_t)stbir__l2_cache_size(); + size_t allowed = budget + budget / 8; + int max_slices = stbir__slice_max_count( output_columns ); + int slices; + + for (slices = 1; slices < max_slices; slices++) + { + int slice_columns = stbir__slice_columns_for( output_columns, slices ); + if ( stbir__ring_bytes_for( horizontal, slice_columns, conservative_in_span, effective_channels, vertical_first, ring_entries ) <= allowed ) + return slices; + } + return max_slices; +} + +// splits the output columns into num_column_slices balanced slices on step boundaries (stbir__slice_end), and works +// out the input range each slice has to decode: the lowest n0 through the furthest pixel any of its kernels read +// (stbir__pack_coefficients arranged for those reads to stay within the conservative range). +static void stbir__build_column_slices( stbir__info * info, int slice_columns, int in_span_bound ) +{ + stbir__sampler const * horizontal = &info->horizontal; + int output_columns = horizontal->scale_info.output_sub_size; + int widest = horizontal->coefficient_width; // the coefficients are packed to widest by now + int first_column, end_column; + int slice_index = 0; + + STBIR__UNUSED( in_span_bound ); // both only checked in asserts + STBIR__UNUSED( slice_columns ); + + for (first_column = 0; first_column < output_columns; first_column = end_column, slice_index++) + { + stbir__column_slice * slice = info->column_slices + slice_index; + int in_first = 0x7fffffff; + int in_last = -0x7fffffff; + int x, unused; + + end_column = stbir__slice_end( output_columns, info->num_column_slices, slice_index ); + STBIR_ASSERT( ( end_column > first_column ) && ( end_column <= output_columns ) && ( ( end_column - first_column ) <= slice_columns ) ); + + // the lowest first pixel through the furthest pixel any of the slice's kernels read + for (x = first_column; x < end_column; x++) + { + stbir__contributors contributor; + int last_read; + + stbir__output_range( horizontal, x, &contributor.n0, &contributor.n1 ); + last_read = contributor.n0 + stbir__horizontal_read_extent( &contributor, widest ) - 1; + if ( contributor.n0 < in_first ) + in_first = contributor.n0; + if ( last_read > in_last ) + in_last = last_read; + } + + STBIR_ASSERT( in_first >= info->conservative_extents.n0 ); + STBIR_ASSERT( in_last <= info->conservative_extents.n1 ); + STBIR_ASSERT( ( in_last - in_first + 1 ) <= in_span_bound ); // the buffers were sized from in_span_bound in the first allocation pass + + slice->x0 = first_column; + slice->width = end_column - first_column; + slice->in_n0 = in_first; + slice->in_n1 = in_last; + stbir__output_range( horizontal, first_column, &slice->first_n0, &unused ); + } + + STBIR_ASSERT( slice_index == info->num_column_slices ); +} + +// split s is row band ( s / split_cols ) of column group ( s % split_cols ). +static void stbir__setup_split_grid( stbir__info * info, int split_cols, int split_rows ) +{ + int r, c, next = 0; + + info->split_cols = split_cols; + info->split_rows = split_rows; + stbir__get_split_info( info->split_info, split_rows, info->vertical.scale_info.output_sub_size, info->vertical.filter_pixel_margin, info->vertical.scale_info.input_full_size, info->vertical.is_gather, &info->vertical ); + for( r = split_rows - 1 ; r >= 0 ; r-- ) + { + stbir__per_split_info band = info->split_info[r]; + for( c = split_cols - 1 ; c >= 0 ; c-- ) + { + stbir__per_split_info * s = info->split_info + r * split_cols + c; + s->start_output_y = band.start_output_y; + s->end_output_y = band.end_output_y; + s->start_input_y = band.start_input_y; + s->end_input_y = band.end_input_y; + } + } + for( c = 0 ; c < split_cols ; c++ ) + { + int count = ( info->num_column_slices - next ) / ( split_cols - c ); + for( r = 0 ; r < split_rows ; r++ ) + { + info->split_info[ r * split_cols + c ].first_column_slice = next; + info->split_info[ r * split_cols + c ].num_column_slices = count; + } + next += count; + } +} + +// the per split buffers get a page of slack, to be repositioned within it at resize time (stbir__4K_alias_skew_buffers) +#define STBIR__4K_ALIAS_SKEW_SLACK 4096 + +// What kind of alpha weighting the layouts call for: 0 = none, 1 = weight in, leave premultiplied, 2 = fancy weight +// in and unweight out, 3 = premultiplied in, unweight out, 4 = simple weight in and unweight out. Fancy weighting +// carries extra channels through the filters (*effective_channels). +static int stbir__alpha_weighting_type( stbir__sampler const * horizontal, stbir__sampler const * vertical, stbir_internal_pixel_layout input_pixel_layout, stbir_internal_pixel_layout output_pixel_layout, int fast_alpha, int * effective_channels ) +{ + int input_has_alpha = ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ); + int input_premultiplied = ( input_pixel_layout >= STBIRI_RGBA_PM ) && ( input_pixel_layout <= STBIRI_AR_PM ); + int output_has_alpha = ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ); + int output_premultiplied = ( output_pixel_layout >= STBIRI_RGBA_PM ) && ( output_pixel_layout <= STBIRI_AR_PM ); + + int point_sampled = ( horizontal->filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( vertical->filter_enum == STBIR_FILTER_POINT_SAMPLE ); + + if ( input_has_alpha && output_has_alpha ) + { + static int fancy_alpha_effective_cnts[6] = { 7, 7, 7, 7, 3, 3 }; + if ( point_sampled ) // nothing is blended, so weighting in and unweighting out would be the identity + return 0; + if ( fast_alpha ) + return 4; + *effective_channels = fancy_alpha_effective_cnts[ input_pixel_layout - STBIRI_RGBA ]; + return 2; + } + // (the conversions between premultiplied and not still happen when point sampling) + if ( input_premultiplied && output_has_alpha ) + return 3; + if ( input_has_alpha && output_premultiplied ) + return 1; + return 0; +} + +// The alpha weight and unweight functions for the weighting type (and the 3 channel color flip, which rides the same path). +static void stbir__set_alpha_functions( stbir__info * info, int alpha_weighting_type, stbir_internal_pixel_layout input_pixel_layout, stbir_internal_pixel_layout output_pixel_layout, float horizontal_scale ) +{ + static stbir__alpha_weight_func * fancy_alpha_weights[6] = { stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_2ch, stbir__fancy_alpha_weight_2ch }; + static stbir__alpha_unweight_func * fancy_alpha_unweights[6] = { stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_2ch, stbir__fancy_alpha_unweight_2ch }; + static stbir__alpha_weight_func * simple_alpha_weights[6] = { stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_2ch, stbir__simple_alpha_weight_2ch }; + static stbir__alpha_unweight_func * simple_alpha_unweights[6] = { stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_2ch, stbir__simple_alpha_unweight_2ch }; + + info->alpha_weight = 0; + info->alpha_unweight = 0; + + if ( alpha_weighting_type == 2 ) + { + // high quality alpha multiplying on the way in, dividing on the way out + info->alpha_weight = fancy_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + info->alpha_unweight = fancy_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 4 ) + { + // fast alpha multiplying on the way in, dividing on the way out + info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 1 ) + { + // fast alpha on the way in, leave in premultiplied form on way out + info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 3 ) + { + // incoming is premultiplied, fast alpha dividing on the way out - non-premultiplied output + info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + + // handle 3-chan color flipping, using the alpha weight path + if ( ( ( input_pixel_layout == STBIRI_RGB ) && ( output_pixel_layout == STBIRI_BGR ) ) || + ( ( input_pixel_layout == STBIRI_BGR ) && ( output_pixel_layout == STBIRI_RGB ) ) ) + { + // do the flipping on the smaller of the two ends + if ( horizontal_scale < 1.0f ) + info->alpha_unweight = stbir__simple_flip_3ch; + else + info->alpha_weight = stbir__simple_flip_3ch; + } +} + +// Shared coefficients require identical geometry; inactive rational and exact-position fields may be uninitialized. +static int stbir__scale_info_matches( stbir__scale_info const * horizontal, stbir__scale_info const * vertical ) +{ + if ( ( horizontal->input_full_size != vertical->input_full_size ) || ( horizontal->output_sub_size != vertical->output_sub_size ) || + ( horizontal->scale != vertical->scale ) || ( horizontal->inv_scale != vertical->inv_scale ) || // (the positions come from the reals) + ( horizontal->pixel_shift != vertical->pixel_shift ) || ( horizontal->scale_is_rational != vertical->scale_is_rational ) || + ( horizontal->positions_exact != vertical->positions_exact ) ) + { + return 0; + } + + if ( horizontal->scale_is_rational && + ( ( horizontal->scale_numerator != vertical->scale_numerator ) || ( horizontal->scale_denominator != vertical->scale_denominator ) ) ) + { + return 0; + } + + if ( horizontal->positions_exact && + ( ( horizontal->shift_pixels != vertical->shift_pixels ) || ( horizontal->tap_scale_in != vertical->tap_scale_in ) || + ( horizontal->tap_scale_out != vertical->tap_scale_out ) ) ) + { + return 0; + } + + return 1; +} + +// Are the two samplers identical? (happens a lot with mipmap generation) Then the vertical can share the horizontal's tables. +static int stbir__samplers_match( stbir__sampler const * horizontal, stbir__sampler const * vertical ) +{ + return ( horizontal->filter_kernel_new == vertical->filter_kernel_new ) && ( horizontal->filter_kernel_old == vertical->filter_kernel_old ) && ( horizontal->filter_support_new == vertical->filter_support_new ) && ( horizontal->filter_radius == vertical->filter_radius ) && ( horizontal->filter_support_old == vertical->filter_support_old ) && + ( horizontal->edge == vertical->edge ) && ( horizontal->is_gather == vertical->is_gather ) && + stbir__scale_info_matches( &horizontal->scale_info, &vertical->scale_info ); +} + +// The horizontal kernels for this channel count and coefficient width. +static void stbir__select_horizontal_kernels( stbir__info * info, stbir__sampler const * horizontal, int effective_channels ) +{ + int widest = horizontal->extent_info.widest; + + // start with defaulting to the n_coeffs functions (specialized on channels and remnant leftover) + info->horizontal_gather_channels = stbir__horizontal_gather_n_coeffs_funcs[ effective_channels ][ widest & 3 ]; + + // but if the number of coeffs <= 17, use another set of special cases. This is by far the common case + if ( widest <= 17 ) + info->horizontal_gather_channels = stbir__horizontal_gather_channels_funcs[ effective_channels ][ widest - 1 ]; + + // integer downsamples (polyphase period 1) use one coefficient set for the whole interior, so the kernel can keep the + // coefficients in registers for the whole run - where there is one for this channel count and width (null otherwise) + info->horizontal_gather_period1 = 0; + if ( ( horizontal->shared_period == 1 ) && ( widest <= 17 ) ) + info->horizontal_gather_period1 = stbir__horizontal_gather_period1_funcs[ effective_channels ][ widest ]; +} + +// The per split buffers are sized for one column slice (the slices run one after another and reuse them). +typedef struct stbir__slice_buffer_sizes +{ + int num_column_slices; // a multiple of the column groups, so they get equal shares + int slice_columns; // the widest slice + int slice_in_span; // the most input pixels a slice decodes + size_t decode_buffer_size; + size_t ring_buffer_length_bytes; // one ring entry + size_t ring_buffer_size; + size_t vertical_buffer_size; +} stbir__slice_buffer_sizes; + +static void stbir__size_slice_buffers( stbir__slice_buffer_sizes * sizes, stbir__sampler const * horizontal, int conservative_in_span, int l2_slices, int split_cols, int effective_channels, int vertical_first, int ring_buffer_num_entries ) +{ + int output_columns = horizontal->scale_info.output_sub_size; + int max_slices = stbir__slice_max_count( output_columns ); + + // the slice count made a multiple of the column groups (a slice is at least a step wide, so some groups may get none) + sizes->num_column_slices = ( ( l2_slices + split_cols - 1 ) / split_cols ) * split_cols; + if ( sizes->num_column_slices > max_slices ) + sizes->num_column_slices = max_slices; + sizes->slice_columns = stbir__slice_columns_for( output_columns, sizes->num_column_slices ); + + // the input pixels a slice decodes: its columns' geometric span, plus the filter width, plus the room + // stbir__pack_coefficients can move a contributor back by and the kernel can read past it (a coefficient + // width each). the exact ranges come from the packed contributors in the second pass, which asserts + // they fit in this. a single slice is exactly the conservative range. + sizes->slice_in_span = stbir__slice_in_span_for( horizontal, sizes->slice_columns, conservative_in_span ); + + // sometimes read one float off in some of the unrolled loops (with a weight of zero coeff, so it doesn't have an effect) + // we use a few extra floats instead of just 1, so that input callback buffer can overlap with the decode buffer without + // the conversion routines overwriting the callback input data. + sizes->decode_buffer_size = (size_t)sizes->slice_in_span * effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for input callback stagger + +#if defined( STBIR__SEPARATE_ALLOCATIONS ) && defined(STBIR_SIMD8) + if ( effective_channels == 3 ) + sizes->decode_buffer_size += sizeof(float); // avx in 3 channel mode needs one float at the start of the buffer (only with separate allocations) +#endif + + sizes->ring_buffer_length_bytes = (size_t)sizes->slice_columns * (size_t)effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for padding + + // if we do vertical first, the ring buffer holds a whole decoded line + if ( vertical_first ) + sizes->ring_buffer_length_bytes = ( sizes->decode_buffer_size + 15 ) & ~15; + + // keep the ring buffer's entries off a page multiple apart: the vertical gather reads taps rows at the same column, + // and at a 4K multiple stride they all fall in the same cache sets + if ( ( sizes->ring_buffer_length_bytes & 4095 ) == 0 ) + sizes->ring_buffer_length_bytes += 64*3; + + sizes->ring_buffer_size = (size_t)ring_buffer_num_entries * sizes->ring_buffer_length_bytes; + + // The vertical buffer holds one horizontally resampled row: gathering, the output row (the vertical gather's result + // horizontal first, the horizontal's result vertical first); scattering, the input row about to be scattered + // (horizontal first) or the output row at eviction (vertical first). The accumulation itself is in the ring. + sizes->vertical_buffer_size = (size_t)sizes->slice_columns * (size_t)effective_channels * sizeof(float) + sizeof(float); // extra float for padding +} + +static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sampler * horizontal, stbir__sampler * vertical, stbir__contributors * conservative, stbir_pixel_layout input_pixel_layout_public, stbir_pixel_layout output_pixel_layout_public, int splits, int new_x, int new_y, int fast_alpha, int input_pixel_bytes, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) { static char stbir_channel_count_index[8]={ 9,0,1,2, 3,9,9,4 }; @@ -7053,172 +8942,123 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample void * alloced = 0; size_t alloced_total = 0; int vertical_first; - size_t decode_buffer_size, ring_buffer_length_bytes, ring_buffer_size, vertical_buffer_size; int alloc_ring_buffer_num_entries; - - int alpha_weighting_type = 0; // 0=none, 1=simple, 2=fancy - int conservative_split_output_size = stbir__get_max_split( splits, vertical->scale_info.output_sub_size ); + int alpha_weighting_type; + int conservative_split_output_size; + int conservative_in_span, l2_slices, split_cols, split_rows; + int copy_horizontal; + stbir__slice_buffer_sizes sizes; stbir_internal_pixel_layout input_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ input_pixel_layout_public ]; stbir_internal_pixel_layout output_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ output_pixel_layout_public ]; int channels = stbir__pixel_channels[ input_pixel_layout ]; int effective_channels = channels; - // first figure out what type of alpha weighting to use (if any) - if ( ( horizontal->filter_enum != STBIR_FILTER_POINT_SAMPLE ) || ( vertical->filter_enum != STBIR_FILTER_POINT_SAMPLE ) ) // no alpha weighting on point sampling - { - if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) ) - { - if ( fast_alpha ) - { - alpha_weighting_type = 4; - } - else - { - static int fancy_alpha_effective_cnts[6] = { 7, 7, 7, 7, 3, 3 }; - alpha_weighting_type = 2; - effective_channels = fancy_alpha_effective_cnts[ input_pixel_layout - STBIRI_RGBA ]; - } - } - else if ( ( input_pixel_layout >= STBIRI_RGBA_PM ) && ( input_pixel_layout <= STBIRI_AR_PM ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) ) - { - // input premult, output non-premult - alpha_weighting_type = 3; - } - else if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA_PM ) && ( output_pixel_layout <= STBIRI_AR_PM ) ) - { - // input non-premult, output premult - alpha_weighting_type = 1; - } - } - // channel in and out count must match currently if ( channels != stbir__pixel_channels[ output_pixel_layout ] ) return 0; - // get vertical first - vertical_first = stbir__should_do_vertical_first( stbir__compute_weights[ (int)stbir_channel_count_index[ effective_channels ] ], horizontal->filter_pixel_width, horizontal->scale_info.scale, horizontal->scale_info.output_sub_size, vertical->filter_pixel_width, vertical->scale_info.scale, vertical->scale_info.output_sub_size, vertical->is_gather, STBIR__V_FIRST_INFO_POINTER ); + alpha_weighting_type = stbir__alpha_weighting_type( horizontal, vertical, input_pixel_layout, output_pixel_layout, fast_alpha, &effective_channels ); - // sometimes read one float off in some of the unrolled loops (with a weight of zero coeff, so it doesn't have an effect) - // we use a few extra floats instead of just 1, so that input callback buffer can overlap with the decode buffer without - // the conversion routines overwriting the callback input data. - decode_buffer_size = ( conservative->n1 - conservative->n0 + 1 ) * effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for input callback stagger + // do the horizontal and vertical store their polyphase coefficient sets once? (sizes the tables, so before anything + // is allocated). the vertical only when gathering (the scatter tables are per input scanline). the vertical kernels + // read exactly n0..n1 of a set, so its epilogue needs no clip zone. + stbir__plan_polyphase_sharing( horizontal, 1, user_data ); + if ( vertical->is_gather ) + stbir__plan_polyphase_sharing( vertical, 0, user_data ); -#if defined( STBIR__SEPARATE_ALLOCATIONS ) && defined(STBIR_SIMD8) - if ( effective_channels == 3 ) - decode_buffer_size += sizeof(float); // avx in 3 channel mode needs one float at the start of the buffer (only with separate allocations) -#endif + vertical_first = stbir__should_do_vertical_first( (int)stbir_channel_count_index[ effective_channels ], + horizontal->filter_pixel_width, horizontal->scale_info.input_full_size, horizontal->scale_info.output_sub_size, horizontal->edge, horizontal->shared_period == 1, + vertical->filter_pixel_width, vertical->scale_info.input_full_size, vertical->scale_info.output_sub_size, vertical->edge ); - ring_buffer_length_bytes = (size_t)horizontal->scale_info.output_sub_size * (size_t)effective_channels * sizeof(float) + sizeof(float)*STBIR_INPUT_CALLBACK_PADDING; // extra floats for padding + // ring rows: gathering, an output's taps rows (plus one for float rounding); scattering, the output rows an input + // row touches - 2 x the filter's support in output pixels, plus one, plus one for rounding + if ( vertical->is_gather ) + alloc_ring_buffer_num_entries = vertical->filter_pixel_width + 1; + else + alloc_ring_buffer_num_entries = stbir__ceil_to_int( (stbir_real)2.0 * stbir__filter_radius_in_output_pixels( vertical->filter_radius, vertical->filter_support_new, vertical->filter_support_old, &vertical->scale_info, user_data ) ) + 2; - // if we do vertical first, the ring buffer holds a whole decoded line - if ( vertical_first ) - ring_buffer_length_bytes = ( decode_buffer_size + 15 ) & ~15; - - if ( ( ring_buffer_length_bytes & 4095 ) == 0 ) ring_buffer_length_bytes += 64*3; // avoid 4k alias - - // One extra entry because floating point precision problems sometimes cause an extra to be necessary. - alloc_ring_buffer_num_entries = vertical->filter_pixel_width + 1; + // how many vertical slices the single threaded resize would run (the setting, or the L2 fit), then the split grid - + // which can lower the split count + conservative_in_span = conservative->n1 - conservative->n0 + 1; + l2_slices = stbir__column_slices; + if ( l2_slices < 1 ) + l2_slices = stbir__choose_column_slices( horizontal, conservative_in_span, effective_channels, vertical_first, alloc_ring_buffer_num_entries ); + stbir__choose_split_grid( &splits, &split_cols, &split_rows, stbir__split_mode, horizontal->scale_info.output_sub_size, vertical->scale_info.output_sub_size, conservative_in_span, input_pixel_bytes, vertical->filter_pixel_width, (float) vertical->scale_info.inv_scale, l2_slices, effective_channels ); + conservative_split_output_size = stbir__get_max_split( split_rows, vertical->scale_info.output_sub_size ); // we never need more ring buffer entries than the scanlines we're outputting when in scatter mode if ( ( !vertical->is_gather ) && ( alloc_ring_buffer_num_entries > conservative_split_output_size ) ) alloc_ring_buffer_num_entries = conservative_split_output_size; - ring_buffer_size = (size_t)alloc_ring_buffer_num_entries * (size_t)ring_buffer_length_bytes; + stbir__size_slice_buffers( &sizes, horizontal, conservative_in_span, l2_slices, split_cols, effective_channels, vertical_first, alloc_ring_buffer_num_entries ); - // The vertical buffer is used differently, depending on whether we are scattering - // the vertical scanlines, or gathering them. - // If scattering, it's used at the temp buffer to accumulate each output. - // If gathering, it's just the output buffer. - vertical_buffer_size = (size_t)horizontal->scale_info.output_sub_size * (size_t)effective_channels * sizeof(float) + sizeof(float); // extra float for padding + // are the two filters identical? then the vertical shares the horizontal's tables instead of generating its own + copy_horizontal = stbir__samplers_match( horizontal, vertical ); // we make two passes through this loop, 1st to add everything up, 2nd to allocate and init for(;;) { int i; void * advance_mem = alloced; - int copy_horizontal = 0; - stbir__sampler * possibly_use_horizontal_for_pivot = 0; #ifdef STBIR__SEPARATE_ALLOCATIONS - #define STBIR__NEXT_PTR( ptr, size, ntype ) if ( alloced ) { void * p = STBIR_MALLOC( size, user_data); if ( p == 0 ) { stbir__free_internal_mem( info ); return 0; } (ptr) = (ntype*)p; } + // a failed allocation frees whatever got allocated: the pointers the cleanup frees are cleared as their holders are + // allocated, so every one is an allocation or null (if info itself failed, only the placeholder block exists) + #define STBIR__NEXT_PTR( ptr, size, ntype ) if ( alloced ) { void * p = STBIR_MALLOC( size, user_data); if ( p == 0 ) { if ( info ) stbir__free_internal_mem( info ); else STBIR_FREE( alloced, user_data ); return 0; } (ptr) = (ntype*)p; } #else #define STBIR__NEXT_PTR( ptr, size, ntype ) advance_mem = (void*) ( ( ((size_t)advance_mem) + 15 ) & ~15 ); if ( alloced ) ptr = (ntype*)advance_mem; advance_mem = (char*)(((size_t)advance_mem) + (size)); #endif STBIR__NEXT_PTR( info, sizeof( stbir__info ), stbir__info ); + if ( info ) + { + info->alloced_mem = alloced; + info->user_data = user_data; + info->splits = splits; + info->alloc_ring_buffer_num_entries = (int)alloc_ring_buffer_num_entries; + info->effective_channels = effective_channels; + info->split_info = 0; + info->column_slices = 0; + info->horizontal.contributors = 0; + info->horizontal.coefficients = 0; + info->vertical.contributors = 0; + info->vertical.coefficients = 0; + } STBIR__NEXT_PTR( info->split_info, sizeof( stbir__per_split_info ) * splits, stbir__per_split_info ); + #ifdef STBIR__SEPARATE_ALLOCATIONS + if ( info ) + { + for( i = 0 ; i < splits ; i++ ) + { + info->split_info[i].ring_buffers = 0; + info->split_info[i].decode_buffer_alloc = 0; + info->split_info[i].vertical_buffer_alloc = 0; + } + } + #endif if ( info ) { - static stbir__alpha_weight_func * fancy_alpha_weights[6] = { stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_2ch, stbir__fancy_alpha_weight_2ch }; - static stbir__alpha_unweight_func * fancy_alpha_unweights[6] = { stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_2ch, stbir__fancy_alpha_unweight_2ch }; - static stbir__alpha_weight_func * simple_alpha_weights[6] = { stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_2ch, stbir__simple_alpha_weight_2ch }; - static stbir__alpha_unweight_func * simple_alpha_unweights[6] = { stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_2ch, stbir__simple_alpha_unweight_2ch }; - // initialize info fields - info->alloced_mem = alloced; info->alloced_total = alloced_total; - info->channels = channels; - info->effective_channels = effective_channels; - info->offset_x = new_x; info->offset_y = new_y; - info->alloc_ring_buffer_num_entries = (int)alloc_ring_buffer_num_entries; info->ring_buffer_num_entries = 0; - info->ring_buffer_length_bytes = (int)ring_buffer_length_bytes; - info->splits = splits; + info->ring_buffer_length_bytes = sizes.ring_buffer_length_bytes; info->vertical_first = vertical_first; info->input_pixel_layout_internal = input_pixel_layout; info->output_pixel_layout_internal = output_pixel_layout; - // setup alpha weight functions - info->alpha_weight = 0; - info->alpha_unweight = 0; - - // handle alpha weighting functions and overrides - if ( alpha_weighting_type == 2 ) - { - // high quality alpha multiplying on the way in, dividing on the way out - info->alpha_weight = fancy_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; - info->alpha_unweight = fancy_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; - } - else if ( alpha_weighting_type == 4 ) - { - // fast alpha multiplying on the way in, dividing on the way out - info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; - info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; - } - else if ( alpha_weighting_type == 1 ) - { - // fast alpha on the way in, leave in premultiplied form on way out - info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; - } - else if ( alpha_weighting_type == 3 ) - { - // incoming is premultiplied, fast alpha dividing on the way out - non-premultiplied output - info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; - } - - // handle 3-chan color flipping, using the alpha weight path - if ( ( ( input_pixel_layout == STBIRI_RGB ) && ( output_pixel_layout == STBIRI_BGR ) ) || - ( ( input_pixel_layout == STBIRI_BGR ) && ( output_pixel_layout == STBIRI_RGB ) ) ) - { - // do the flipping on the smaller of the two ends - if ( horizontal->scale_info.scale < 1.0f ) - info->alpha_unweight = stbir__simple_flip_3ch; - else - info->alpha_weight = stbir__simple_flip_3ch; - } - + stbir__set_alpha_functions( info, alpha_weighting_type, input_pixel_layout, output_pixel_layout, (float) horizontal->scale_info.scale ); } - // get all the per-split buffers + // get all the per-split buffers (each with a page of slack, see stbir__4K_alias_skew_buffers) for( i = 0 ; i < splits ; i++ ) { - STBIR__NEXT_PTR( info->split_info[i].decode_buffer, decode_buffer_size, float ); + STBIR__NEXT_PTR( info->split_info[i].decode_buffer, sizes.decode_buffer_size + STBIR__4K_ALIAS_SKEW_SLACK, float ); #ifdef STBIR__SEPARATE_ALLOCATIONS @@ -7226,13 +9066,20 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample if ( ( info ) && ( effective_channels == 3 ) ) ++info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer #endif + if ( info ) + info->split_info[i].decode_buffer_alloc = info->split_info[i].decode_buffer; STBIR__NEXT_PTR( info->split_info[i].ring_buffers, alloc_ring_buffer_num_entries * sizeof(float*), float* ); { int j; + if ( info ) + { + for( j = 0 ; j < alloc_ring_buffer_num_entries ; j++ ) + info->split_info[i].ring_buffers[j] = 0; + } for( j = 0 ; j < alloc_ring_buffer_num_entries ; j++ ) { - STBIR__NEXT_PTR( info->split_info[i].ring_buffers[j], ring_buffer_length_bytes, float ); + STBIR__NEXT_PTR( info->split_info[i].ring_buffers[j], sizes.ring_buffer_length_bytes, float ); #ifdef STBIR_SIMD8 if ( ( info ) && ( effective_channels == 3 ) ) ++info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer @@ -7240,118 +9087,86 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample } } #else - STBIR__NEXT_PTR( info->split_info[i].ring_buffer, ring_buffer_size, float ); + STBIR__NEXT_PTR( info->split_info[i].ring_buffer, sizes.ring_buffer_size + STBIR__4K_ALIAS_SKEW_SLACK, float ); #endif - STBIR__NEXT_PTR( info->split_info[i].vertical_buffer, vertical_buffer_size, float ); + STBIR__NEXT_PTR( info->split_info[i].vertical_buffer, sizes.vertical_buffer_size + STBIR__4K_ALIAS_SKEW_SLACK, float ); + + if ( info ) + { + #ifndef STBIR__SEPARATE_ALLOCATIONS + info->split_info[i].decode_buffer_alloc = info->split_info[i].decode_buffer; + info->split_info[i].ring_buffer_alloc = info->split_info[i].ring_buffer; + #endif + info->split_info[i].vertical_buffer_alloc = info->split_info[i].vertical_buffer; + } } - // alloc memory for to-be-pivoted coeffs (if necessary) - if ( vertical->is_gather == 0 ) + // the tables are allocated into info's samplers, and the samplers are built in the locals (copied into info below) + STBIR__NEXT_PTR( info->horizontal.contributors, horizontal->contributors_size, stbir__contributors ); + STBIR__NEXT_PTR( info->horizontal.coefficients, horizontal->coefficients_size, float ); + + if ( !copy_horizontal ) // (sharing: the vertical reads the horizontal's tables, see below) { - size_t both; - size_t temp_mem_amt; - - // when in vertical scatter mode, we first build the coefficients in gather mode, and then pivot after, - // that means we need two buffers, so we try to use the decode buffer and ring buffer for this. if that - // is too small, we just allocate extra memory to use as this temp. - - both = (size_t)vertical->gather_prescatter_contributors_size + (size_t)vertical->gather_prescatter_coefficients_size; - -#ifdef STBIR__SEPARATE_ALLOCATIONS - temp_mem_amt = decode_buffer_size; - - #ifdef STBIR_SIMD8 - if ( effective_channels == 3 ) - --temp_mem_amt; // avx in 3 channel mode needs one float at the start of the buffer - #endif -#else - temp_mem_amt = (size_t)( decode_buffer_size + ring_buffer_size + vertical_buffer_size ) * (size_t)splits; -#endif - if ( temp_mem_amt >= both ) - { - if ( info ) - { - vertical->gather_prescatter_contributors = (stbir__contributors*)info->split_info[0].decode_buffer; - vertical->gather_prescatter_coefficients = (float*) ( ( (char*)info->split_info[0].decode_buffer ) + vertical->gather_prescatter_contributors_size ); - } - } - else - { - // ring+decode memory is too small, so allocate temp memory - STBIR__NEXT_PTR( vertical->gather_prescatter_contributors, vertical->gather_prescatter_contributors_size, stbir__contributors ); - STBIR__NEXT_PTR( vertical->gather_prescatter_coefficients, vertical->gather_prescatter_coefficients_size, float ); - } + STBIR__NEXT_PTR( info->vertical.contributors, vertical->contributors_size, stbir__contributors ); + STBIR__NEXT_PTR( info->vertical.coefficients, vertical->coefficients_size, float ); } - STBIR__NEXT_PTR( horizontal->contributors, horizontal->contributors_size, stbir__contributors ); - STBIR__NEXT_PTR( horizontal->coefficients, horizontal->coefficients_size, float ); - - // are the two filters identical?? (happens a lot with mipmap generation) - if ( ( horizontal->filter_kernel == vertical->filter_kernel ) && ( horizontal->filter_support == vertical->filter_support ) && ( horizontal->edge == vertical->edge ) && ( horizontal->scale_info.output_sub_size == vertical->scale_info.output_sub_size ) ) + if ( info ) { - float diff_scale = horizontal->scale_info.scale - vertical->scale_info.scale; - float diff_shift = horizontal->scale_info.pixel_shift - vertical->scale_info.pixel_shift; - if ( diff_scale < 0.0f ) diff_scale = -diff_scale; - if ( diff_shift < 0.0f ) diff_shift = -diff_shift; - if ( ( diff_scale <= stbir__small_float ) && ( diff_shift <= stbir__small_float ) ) - { - if ( horizontal->is_gather == vertical->is_gather ) - { - copy_horizontal = 1; - goto no_vert_alloc; - } - // everything matches, but vertical is scatter, horizontal is gather, use horizontal coeffs for vertical pivot coeffs - possibly_use_horizontal_for_pivot = horizontal; - } + horizontal->contributors = info->horizontal.contributors; + horizontal->coefficients = info->horizontal.coefficients; + vertical->contributors = info->vertical.contributors; + vertical->coefficients = info->vertical.coefficients; } - STBIR__NEXT_PTR( vertical->contributors, vertical->contributors_size, stbir__contributors ); - STBIR__NEXT_PTR( vertical->coefficients, vertical->coefficients_size, float ); - - no_vert_alloc: + STBIR__NEXT_PTR( info->column_slices, sizeof( stbir__column_slice ) * sizes.num_column_slices, stbir__column_slice ); if ( info ) { STBIR_PROFILE_BUILD_START( horizontal ); - stbir__calculate_filters( horizontal, 0, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + stbir__calculate_filters( horizontal, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + stbir__select_horizontal_kernels( info, horizontal, effective_channels ); - // setup the horizontal gather functions - // start with defaulting to the n_coeffs functions (specialized on channels and remnant leftover) - info->horizontal_gather_channels = stbir__horizontal_gather_n_coeffs_funcs[ effective_channels ][ horizontal->extent_info.widest & 3 ]; - // but if the number of coeffs <= 12, use another set of special cases. <=12 coeffs is any enlarging resize, or shrinking resize down to about 1/3 size - if ( horizontal->extent_info.widest <= 12 ) - info->horizontal_gather_channels = stbir__horizontal_gather_channels_funcs[ effective_channels ][ horizontal->extent_info.widest - 1 ]; + info->conservative_extents = *conservative; - info->scanline_extents.conservative.n0 = conservative->n0; - info->scanline_extents.conservative.n1 = conservative->n1; - - // get exact extents - stbir__get_extents( horizontal, &info->scanline_extents ); - - // pack the horizontal coeffs - horizontal->coefficient_width = stbir__pack_coefficients(horizontal->num_contributors, horizontal->contributors, horizontal->coefficients, horizontal->coefficient_width, horizontal->extent_info.widest, info->scanline_extents.conservative.n0, info->scanline_extents.conservative.n1 ); + // pack the horizontal coeffs into the column entries the kernels walk (the vertical's aren't packed: its kernels read exactly n0..n1) + if ( horizontal->shared_period ) + horizontal->coefficient_width = stbir__pack_shared_coefficients( horizontal, info->conservative_extents.n0, info->conservative_extents.n1 ); + else + horizontal->coefficient_width = stbir__pack_coefficients(horizontal->num_contributors, horizontal->contributors, horizontal->coefficients, horizontal->coefficient_width, horizontal->extent_info.widest, info->conservative_extents.n0, info->conservative_extents.n1 ); + horizontal->entry_floats = STBIR__COLUMN_ENTRY_FLOATS( horizontal->coefficient_width ); STBIR_MEMCPY( &info->horizontal, horizontal, sizeof( stbir__sampler ) ); + // now that the contributors are final, work out exactly what each column slice decodes + info->num_column_slices = sizes.num_column_slices; + stbir__build_column_slices( info, sizes.slice_columns, sizes.slice_in_span ); + STBIR_PROFILE_BUILD_END( horizontal ); + STBIR_PROFILE_BUILD_START( vertical ); if ( copy_horizontal ) { - STBIR_MEMCPY( &info->vertical, horizontal, sizeof( stbir__sampler ) ); + // identical samplers: the vertical reads the horizontal's packed column entries (two ints, then the coefficients) + STBIR_MEMCPY( vertical, horizontal, sizeof( stbir__sampler ) ); + vertical->coefficients += 2; + vertical->coefficient_width = horizontal->entry_floats; } else + stbir__calculate_filters( vertical, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + STBIR_MEMCPY( &info->vertical, vertical, sizeof( stbir__sampler ) ); + STBIR_PROFILE_BUILD_END( vertical ); + + // the horizontal's contributors have been read for the last time (the slice table), so fill in the column entries' + // advance and width (an advance that doesn't fit fails the build) + if ( !stbir__fill_column_entries( &info->horizontal, effective_channels ) ) { - STBIR_PROFILE_BUILD_START( vertical ); - - stbir__calculate_filters( vertical, possibly_use_horizontal_for_pivot, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); - STBIR_MEMCPY( &info->vertical, vertical, sizeof( stbir__sampler ) ); - - STBIR_PROFILE_BUILD_END( vertical ); + stbir__free_internal_mem( info ); + return 0; } - // setup the vertical split ranges - stbir__get_split_info( info->split_info, info->splits, info->vertical.scale_info.output_sub_size, info->vertical.filter_pixel_margin, info->vertical.scale_info.input_full_size, info->vertical.is_gather, info->vertical.contributors ); + stbir__setup_split_grid( info, split_cols, split_rows ); // now we know precisely how many entries we need info->ring_buffer_num_entries = info->vertical.extent_info.widest; @@ -7363,7 +9178,6 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample } #undef STBIR__NEXT_PTR - // is this the first time through loop? if ( info == 0 ) { @@ -7377,17 +9191,76 @@ static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sample } } +// 4K aliasing avoidance: each internal buffer has a page of slack and is placed half a page from the buffer +// that streams into it - input -> decode -> ring -> vertical buffer (horizontal first), +// input -> ring -> decode -> vertical buffer (vertical first). +static size_t stbir__4K_alias_place( void * alloc, size_t want ) +{ + size_t have = ( (size_t)alloc ) & 4095; + return ( ( want - have ) & 4095 ) & ~(size_t)15; // stays 16 byte aligned, within the slack +} + +static void stbir__4K_alias_skew_buffers( stbir__info const * info, stbir__per_split_info * split_info ) +{ + size_t first = ( ( (size_t)info->input_data ) + 2048 ) & 4095; // the first internal buffer: half a page from the input row + size_t second = ( first + 2048 ) & 4095; // and the next one, back at the input's offset + size_t vertical_dest; + + if ( info->vertical_first ) + { + // decode: input -> ring. vertical: ring -> decode_buffer. horizontal: decode_buffer -> vertical_buffer. encode: vertical_buffer -> output + #ifndef STBIR__SEPARATE_ALLOCATIONS + split_info->ring_buffer = (float*) ( (char*)split_info->ring_buffer_alloc + stbir__4K_alias_place( split_info->ring_buffer_alloc, first ) ); + #endif + vertical_dest = ( second + (size_t)stbir__4K_vertical_dest_shift ) & 4095; + split_info->decode_buffer = (float*) ( (char*)split_info->decode_buffer_alloc + stbir__4K_alias_place( split_info->decode_buffer_alloc, vertical_dest ) ); + split_info->vertical_buffer = (float*) ( (char*)split_info->vertical_buffer_alloc + stbir__4K_alias_place( split_info->vertical_buffer_alloc, ( vertical_dest + 2048 ) & 4095 ) ); + } + else + { + // decode: input -> decode_buffer. horizontal: decode_buffer -> ring. vertical: ring -> vertical_buffer. encode: vertical_buffer -> output + split_info->decode_buffer = (float*) ( (char*)split_info->decode_buffer_alloc + stbir__4K_alias_place( split_info->decode_buffer_alloc, first ) ); + #ifndef STBIR__SEPARATE_ALLOCATIONS + split_info->ring_buffer = (float*) ( (char*)split_info->ring_buffer_alloc + stbir__4K_alias_place( split_info->ring_buffer_alloc, second ) ); + #endif + vertical_dest = ( first + (size_t)stbir__4K_vertical_dest_shift ) & 4095; + split_info->vertical_buffer = (float*) ( (char*)split_info->vertical_buffer_alloc + stbir__4K_alias_place( split_info->vertical_buffer_alloc, vertical_dest ) ); + } +} + static int stbir__perform_resize( stbir__info const * info, int split_start, int split_count ) { stbir__per_split_info * split_info = info->split_info + split_start; + int b, i, can_merge; + + for( i = 0 ; i < split_count ; i++ ) + stbir__4K_alias_skew_buffers( info, split_info + i ); STBIR_PROFILE_CLEAR_EXTRAS(); STBIR_PROFILE_FIRST_START( looping ); - if (info->vertical.is_gather) - stbir__vertical_gather_loop( info, split_info, split_count ); - else - stbir__vertical_scatter_loop( info, split_info, split_count ); + // run the whole vertical loop once per column slice, so each pass only touches a cache sized slice of the buffers. + // consecutive splits over the same slices are adjacent row bands and run as one band (one warm-up per slice) - except + // when a scatter's ring was cut down to a band's rows: an input row can touch more output rows than that across + // two bands, so those run one at a time + can_merge = info->vertical.is_gather || ( info->ring_buffer_num_entries >= info->vertical.extent_info.widest ); + i = 0; + while ( i < split_count ) + { + int run = 1; + while ( can_merge && ( i + run < split_count ) && ( split_info[i+run].first_column_slice == split_info[i].first_column_slice ) && ( split_info[i+run].num_column_slices == split_info[i].num_column_slices ) ) + ++run; + STBIR_PROFILE_NEST_IN_FIRST( split_info + i ); + for( b = 0 ; b < split_info[i].num_column_slices ; b++ ) + { + stbir__column_slice const * slice = info->column_slices + split_info[i].first_column_slice + b; + if (info->vertical.is_gather) + stbir__vertical_gather_loop( info, split_info + i, run, slice ); + else + stbir__vertical_scatter_loop( info, split_info + i, run, slice ); + } + i += run; + } STBIR_PROFILE_END( looping ); return 1; @@ -7458,6 +9331,7 @@ static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * r stbir__decode_pixels_func * decode_pixels = 0; stbir__encode_pixels_func * encode_pixels = 0; stbir_datatype input_type, output_type; + int non_scaled; input_type = resize->input_data_type; output_type = resize->output_data_type; @@ -7476,30 +9350,30 @@ static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * r } } - // recalc the output and input strides + // recalc the output and input strides (zero means packed - the whole image's width, the sub rect is a region of it) if ( info->input_stride_bytes == 0 ) - info->input_stride_bytes = info->channels * info->horizontal.scale_info.input_full_size * stbir__type_size[input_type]; + info->input_stride_bytes = (stbir_int64)info->channels * info->horizontal.scale_info.input_full_size * stbir__type_size[input_type]; if ( info->output_stride_bytes == 0 ) - info->output_stride_bytes = info->channels * info->horizontal.scale_info.output_sub_size * stbir__type_size[output_type]; + info->output_stride_bytes = (stbir_int64)info->channels * resize->output_w * stbir__type_size[output_type]; // calc offset - info->output_data = ( (char*) resize->output_pixels ) + ( (size_t) info->offset_y * (size_t) resize->output_stride_in_bytes ) + ( info->offset_x * info->channels * stbir__type_size[output_type] ); + info->output_data = ( (char*) resize->output_pixels ) + (ptrdiff_t)( info->offset_y * info->output_stride_bytes ) + ( (size_t) info->offset_x * (size_t) info->channels * (size_t) stbir__type_size[output_type] ); info->in_pixels_cb = resize->input_cb; info->user_data = resize->user_data; info->out_pixels_cb = resize->output_cb; + // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16) + // don't short circuit when alpha weighting (get everything to 0-1.0 as usual) + non_scaled = 0; + if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) + if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) ) + non_scaled = 1; + // setup the input format converters if ( ( input_type == STBIR_TYPE_UINT8 ) || ( input_type == STBIR_TYPE_UINT16 ) ) { - int non_scaled = 0; - - // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16) - if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual) - if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) ) - non_scaled = 1; - if ( info->input_pixel_layout_internal <= STBIRI_4CHANNEL ) decode_pixels = decode_simple_scaled_or_not[ input_type == STBIR_TYPE_UINT16 ][ non_scaled ]; else @@ -7516,13 +9390,6 @@ static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * r // setup the output format converters if ( ( output_type == STBIR_TYPE_UINT8 ) || ( output_type == STBIR_TYPE_UINT16 ) ) { - int non_scaled = 0; - - // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16) - if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual) - if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) ) - non_scaled = 1; - if ( info->output_pixel_layout_internal <= STBIRI_4CHANNEL ) encode_pixels = encode_simple_scaled_or_not[ output_type == STBIR_TYPE_UINT16 ][ non_scaled ]; else @@ -7540,19 +9407,24 @@ static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * r info->output_type = output_type; info->decode_pixels = decode_pixels; info->encode_pixels = encode_pixels; + + // the input callback's temp buffer sits inside the decode buffer, pushed forward so the decoder reads ahead of + // its writes - except plain float, which is a no-op decode when the temp is exactly the decode target + info->input_callback_offset_bytes = ( decode_pixels == stbir__decode_float_linear ) ? 0 : ( sizeof(float) * STBIR_INPUT_CALLBACK_PADDING ); } -static void stbir__clip( int * outx, int * outsubw, int outw, double * u0, double * u1 ) +static void stbir__clip( int * outx, int * outsubw, int outw, stbir_real * u0, stbir_real * u1 ) { - double per, adj; + stbir_real per, adj; int over; // do left/top edge if ( *outx < 0 ) { - per = ( (double)*outx ) / ( (double)*outsubw ); // is negative + per = ( (stbir_real)*outx ) / ( (stbir_real)*outsubw ); // is negative adj = per * ( *u1 - *u0 ); *u0 -= adj; // increases u0 + *outsubw += *outx; // and drop the clipped off pixels from the output count *outx = 0; } @@ -7560,133 +9432,259 @@ static void stbir__clip( int * outx, int * outsubw, int outw, double * u0, doubl over = outw - ( *outx + *outsubw ); if ( over < 0 ) { - per = ( (double)over ) / ( (double)*outsubw ); // is negative + per = ( (stbir_real)over ) / ( (stbir_real)*outsubw ); // is negative adj = per * ( *u1 - *u0 ); *u1 += adj; // decrease u1 *outsubw = outw - *outx; } } -// converts a double to a rational that has less than one float bit of error (returns 0 if unable to do so) -static int stbir__double_to_rational(double f, stbir_uint32 limit, stbir_uint32 *numer, stbir_uint32 *denom, int limit_denom ) // limit_denom (1) or limit numer (0) +static stbir_uint64 stbir__gcd( stbir_uint64 a, stbir_uint64 b ) { - double err; - stbir_uint64 top, bot; - stbir_uint64 numer_last = 0; - stbir_uint64 denom_last = 1; - stbir_uint64 numer_estimate = 1; - stbir_uint64 denom_estimate = 0; - - // scale to past float error range - top = (stbir_uint64)( f * (double)(1 << 25) ); - bot = 1 << 25; - - // keep refining, but usually stops in a few loops - usually 5 for bad cases - for(;;) + while ( b ) { - stbir_uint64 est, temp; - - // hit limit, break out and do best full range estimate - if ( ( ( limit_denom ) ? denom_estimate : numer_estimate ) >= limit ) - break; - - // is the current error less than 1 bit of a float? if so, we're done - if ( denom_estimate ) - { - err = ( (double)numer_estimate / (double)denom_estimate ) - f; - if ( err < 0.0 ) err = -err; - if ( err < ( 1.0 / (double)(1<<24) ) ) - { - // yup, found it - *numer = (stbir_uint32) numer_estimate; - *denom = (stbir_uint32) denom_estimate; - return 1; - } - } - - // no more refinement bits left? break out and do full range estimate - if ( bot == 0 ) - break; - - // gcd the estimate bits - est = top / bot; - temp = top % bot; - top = bot; - bot = temp; - - // move remainders - temp = est * denom_estimate + denom_last; - denom_last = denom_estimate; - denom_estimate = temp; - - // move remainders - temp = est * numer_estimate + numer_last; - numer_last = numer_estimate; - numer_estimate = temp; + stbir_uint64 remainder = a % b; + a = b; + b = remainder; } - - // we didn't find anything good enough for float, use a full range estimate - if ( limit_denom ) - { - numer_estimate= (stbir_uint64)( f * (double)limit + 0.5 ); - denom_estimate = limit; - } - else - { - numer_estimate = limit; - denom_estimate = (stbir_uint64)( ( (double)limit / f ) + 0.5 ); - } - - *numer = (stbir_uint32) numer_estimate; - *denom = (stbir_uint32) denom_estimate; - - err = ( denom_estimate ) ? ( ( (double)(stbir_uint32)numer_estimate / (double)(stbir_uint32)denom_estimate ) - f ) : 1.0; - if ( err < 0.0 ) err = -err; - return ( err < ( 1.0 / (double)(1<<24) ) ) ? 1 : 0; + return a; } -static int stbir__calculate_region_transform( stbir__scale_info * scale_info, int output_full_range, int * output_offset, int output_sub_range, int input_full_range, double input_s0, double input_s1 ) +// The simplest fraction within tolerance of f (f>0): its continued fraction's first convergent that is, in 31 bits (or 0). +static int stbir__real_to_fraction( stbir_real f, stbir_real tolerance, stbir_uint32 * out_numer, stbir_uint32 * out_denom ) { - double output_range, input_range, output_s, input_s, ratio, scale; + stbir_real rem = f; + stbir_uint64 numer = 1, denom = 0, numer_last = 0, denom_last = 1; // the convergents numer / denom + int i; + + if ( f <= (stbir_real) 0.0 ) + return 0; + for( i = 0 ; i < (int) sizeof( stbir_real ) * 8 ; i++ ) // (a term is at least a bit of f) + { + stbir_uint64 term = (stbir_uint64) rem; // the floor (rem is positive) + stbir_uint64 next; + stbir_real frac, err; + + if ( ( term > 0x7fffffff ) || ( ( term * numer + numer_last ) > 0x7fffffff ) || ( ( term * denom + denom_last ) > 0x7fffffff ) ) + return 0; + next = term * numer + numer_last; numer_last = numer; numer = next; + next = term * denom + denom_last; denom_last = denom; denom = next; + + err = (stbir_real) numer / (stbir_real) denom - f; + if ( err < (stbir_real) 0.0 ) + err = -err; + if ( err <= tolerance ) + { + *out_numer = (stbir_uint32) numer; + *out_denom = (stbir_uint32) denom; + return 1; + } + + frac = rem - (stbir_real) term; + if ( frac <= (stbir_real) 0.0 ) + return 0; // (the fraction is exactly f, and still not within tolerance - can't happen for a tolerance >= 0) + rem = (stbir_real) 1.0 / frac; + } + return 0; +} + +// try to find approximation that stays under the drift value: the far edge of the output moves by less than +// 1 / STBIR__POLYPHASE_DRIFT_DIV of an output pixel +#define STBIR__POLYPHASE_DRIFT_DIV 8192 +static void stbir__approximate_polyphase( stbir_uint64 numer, stbir_uint64 denom, stbir_uint64 outputs, stbir_uint64 * out_numer, stbir_uint64 * out_denom, int * out_drift_in_pixels ) +{ + stbir_uint64 top = numer, bot = denom; + stbir_uint64 p = 1, q = 0, p_last = 0, q_last = 1; + + while ( bot ) + { + stbir_uint64 est = top / bot; + stbir_uint64 remainder = top % bot; + stbir_uint64 next, pd, qn, diff; + + top = bot; + bot = remainder; + + // the next convergent p / q + next = est * p + p_last; + p_last = p; + p = next; + + next = est * q + q_last; + q_last = q; + q = next; + + // no simpler fraction is close enough (the convergents only grow from here) + if ( p >= outputs ) + return; + + // its drift at the far edge, in output pixels: outputs x | p D - q N | / ( q N ) + pd = p * denom; + qn = q * numer; + diff = ( pd > qn ) ? pd - qn : qn - pd; + if ( diff <= qn / ( outputs * STBIR__POLYPHASE_DRIFT_DIV ) ) + { + // that drift at the far edge in input pixels (times denom / numer), rounded up - for the conservative range + stbir_real drift_in = (stbir_real)outputs * (stbir_real)diff * (stbir_real)denom / ( (stbir_real)qn * (stbir_real)numer ); + *out_numer = p; + *out_denom = q; + *out_drift_in_pixels = (int)drift_in + 1; + return; + } + } +} + +// The scale as a fraction: output_sub / ( input_full x ( s1 - s0 ) ), with s1 - s0 = ( n1 - n0 ) / d - exact if possible. +// Returns 0 when there is no fraction that fits in int. +static int stbir__scale_to_rational( int output_sub_range, int input_full_range, stbir_real input_s, int s_n0, int s_n1, int s_d, stbir_uint32 * out_numer, stbir_uint32 * out_denom, int * out_approximated_drift ) +{ + stbir_uint64 numer, denom, g; + stbir_real scale, err; + + *out_numer = 0; // (not a fraction: the fields are read before the flag in places, so never leave them unset) + *out_denom = 0; + *out_approximated_drift = 0; + + if ( ( s_d > 0 ) && ( s_n1 > s_n0 ) ) + { + numer = (stbir_uint64)output_sub_range * (stbir_uint64)s_d; + denom = (stbir_uint64)input_full_range * (stbir_uint64)( s_n1 - s_n0 ); + g = stbir__gcd( numer, denom ); + numer /= g; denom /= g; + if ( ( numer <= 0x7fffffff ) && ( denom <= 0x7fffffff ) ) + { + if ( numer >= (stbir_uint64)output_sub_range ) + stbir__approximate_polyphase( numer, denom, (stbir_uint64)output_sub_range, &numer, &denom, out_approximated_drift ); + *out_numer = (stbir_uint32) numer; + *out_denom = (stbir_uint32) denom; + return 1; + } + } + + scale = (stbir_real)output_sub_range / ( (stbir_real)input_full_range * input_s ); + if ( !stbir__real_to_fraction( scale, scale / ( (stbir_real)output_sub_range * (stbir_real)STBIR__POLYPHASE_DRIFT_DIV ), out_numer, out_denom ) ) + return 0; + err = (stbir_real)*out_numer / (stbir_real)*out_denom - scale; + if ( err < (stbir_real) 0.0 ) + err = -err; + if ( err > (stbir_real) 0.0 ) + *out_approximated_drift = (int)( (stbir_real)output_sub_range * err / ( scale * scale ) ) + 1; // the far edge's drift: output pixels x ( err / scale ), in input pixels ( / scale ) + return 1; +} + +// clip the input area to the image (and adjust the output area), the counterpart of stbir__clip: the output pixels whose +// centers land outside the image are dropped from the ends, so the ones left sample exactly where they did +static void stbir__clip_input( int * outx, int * outsubw, stbir_real * u0, stbir_real * u1 ) +{ + stbir_real per_pixel = ( *u1 - *u0 ) / ( (stbir_real)*outsubw ); // input area per output pixel + int drop; + + // do left/top edge + if ( *u0 < 0 ) + { + drop = stbir__ceil_to_int( -*u0 / per_pixel - (stbir_real)0.5 ); // pixels centered before the image (one right on the edge stays) + if ( drop > *outsubw ) + drop = *outsubw; + if ( drop > 0 ) + { + *u0 += (stbir_real)drop * per_pixel; // increases u0 + *outx += drop; + *outsubw -= drop; + } + } + + // do right/bot edge + if ( *u1 > 1 ) + { + drop = stbir__ceil_to_int( ( *u1 - 1 ) / per_pixel - (stbir_real)0.5 ); // pixels centered past the image + if ( drop > *outsubw ) + drop = *outsubw; + if ( drop > 0 ) + { + *u1 -= (stbir_real)drop * per_pixel; // decreases u1 + *outsubw -= drop; + } + } +} + +static int stbir__calculate_region_transform( stbir__scale_info * scale_info, int output_full_range, int * output_offset, int output_sub_range, int input_full_range, stbir_real input_s0, stbir_real input_s1, int s_n0, int s_n1, int s_d ) +{ + stbir_real output_range, input_range, output_s, input_s, input_pixels, ratio, unclipped_input_s; + int unclipped_output_sub_range = output_sub_range; + int unclipped_output_offset = *output_offset; input_s = input_s1 - input_s0; + unclipped_input_s = input_s; - // null area - if ( ( output_full_range == 0 ) || ( input_full_range == 0 ) || - ( output_sub_range == 0 ) || ( input_s <= stbir__small_float ) ) + // null area (a negative size is an empty one) + if ( ( output_full_range <= 0 ) || ( input_full_range <= 0 ) || + ( output_sub_range <= 0 ) || ( input_s <= (stbir_real)0.0 ) ) return 0; // are either of the ranges completely out of bounds? - if ( ( *output_offset >= output_full_range ) || ( ( *output_offset + output_sub_range ) <= 0 ) || ( input_s0 >= (1.0f-stbir__small_float) ) || ( input_s1 <= stbir__small_float ) ) + if ( ( *output_offset >= output_full_range ) || ( ( *output_offset + output_sub_range ) <= 0 ) || ( input_s0 >= (stbir_real)1.0 ) || ( input_s1 <= (stbir_real)0.0 ) ) return 0; - output_range = (double)output_full_range; - input_range = (double)input_full_range; + output_range = (stbir_real)output_full_range; + input_range = (stbir_real)input_full_range; - output_s = ( (double)output_sub_range) / output_range; + output_s = ( (stbir_real)output_sub_range) / output_range; // figure out the scaling to use ratio = output_s / input_s; - // save scale before clipping - scale = ( output_range / input_range ) * ratio; - scale_info->scale = (float)scale; - scale_info->inv_scale = (float)( 1.0 / scale ); + // save scale before clipping: the output pixels over the input pixels the region spans, and its inverse as the same + // two numbers divided the other way - one rounding each, and since a division is monotonic the two agree about 1 + // (a scale that rounds to exactly 1 can't have an inverse over 1, which would put a ratio over 1 to the kernels) + input_pixels = input_range * input_s; + scale_info->scale = (stbir_real)output_sub_range / input_pixels; + scale_info->inv_scale = input_pixels / (stbir_real)output_sub_range; // clip output area to left/right output edges (and adjust input area) stbir__clip( output_offset, &output_sub_range, output_full_range, &input_s0, &input_s1 ); + // clip input area to the image (and adjust output area) + stbir__clip_input( output_offset, &output_sub_range, &input_s0, &input_s1 ); + // recalc input area input_s = input_s1 - input_s0; // after clipping do we have zero input area? - if ( input_s <= stbir__small_float ) + if ( ( input_s <= (stbir_real)0.0 ) || ( output_sub_range <= 0 ) ) return 0; // calculate and store the starting source offsets in output pixel space - scale_info->pixel_shift = (float) ( input_s0 * ratio * output_range ); + scale_info->pixel_shift = input_s0 * ratio * output_range; - scale_info->scale_is_rational = stbir__double_to_rational( scale, ( scale <= 1.0 ) ? output_full_range : input_full_range, &scale_info->scale_numerator, &scale_info->scale_denominator, ( scale >= 1.0 ) ); + scale_info->scale_is_rational = stbir__scale_to_rational( unclipped_output_sub_range, input_full_range, unclipped_input_s, s_n0, s_n1, s_d, &scale_info->scale_numerator, &scale_info->scale_denominator, &scale_info->approximated_drift ); + + // an exact fraction gives the scale itself - correctly rounded, so an exact ratio's filter widths ( ceil( 2 support / scale ) ) + // don't pick up an extra tap from the region math landing an ulp under the integer, and a 1:1 is exactly 1 (the up / down + // decision). an approximated fraction leaves the real scale: it is what the kernels see, and the decision has to agree with it + if ( ( scale_info->scale_is_rational ) && ( !scale_info->approximated_drift ) ) + { + scale_info->scale = (stbir_real) scale_info->scale_numerator / (stbir_real) scale_info->scale_denominator; + scale_info->inv_scale = (stbir_real) scale_info->scale_denominator / (stbir_real) scale_info->scale_numerator; + } + + // exact tap positions when the shift is an integer ( n0 x output_sub / ( n1 - n0 ), plus the clip ) and the fraction wasn't approximated. + // the numerators are ( 2o + 1 + 2 shift ) D and ( 2i + 1 ) N: with sizes up to 2^30 (the resizer's limit) and the + // fraction's terms below 2^30 they stay below 2^62 + scale_info->positions_exact = 0; + scale_info->shift_pixels = 0; + if ( ( scale_info->scale_is_rational ) && ( !scale_info->approximated_drift ) && ( s_d > 0 ) && ( ( ( (stbir_int64)s_n0 * unclipped_output_sub_range ) % ( s_n1 - s_n0 ) ) == 0 ) && + ( scale_info->scale_numerator < ( 1 << 30 ) ) && ( scale_info->scale_denominator < ( 1 << 30 ) ) ) + { + stbir_int64 shift = ( (stbir_int64)s_n0 * unclipped_output_sub_range ) / ( s_n1 - s_n0 ) + ( *output_offset - unclipped_output_offset ); + if ( ( shift > -( 1 << 30 ) ) && ( shift < ( 1 << 30 ) ) ) // the shift is in output pixels too (an output tile far past the image's own size) + { + scale_info->shift_pixels = (int) shift; + scale_info->tap_scale_in = (stbir_real) 1.0 / ( (stbir_real) 2.0 * (stbir_real) scale_info->scale_numerator ); + scale_info->tap_scale_out = (stbir_real) 1.0 / ( (stbir_real) 2.0 * (stbir_real) scale_info->scale_denominator ); + scale_info->pixel_shift = (stbir_real) shift; + scale_info->positions_exact = 1; + } + } scale_info->input_full_size = input_full_range; scale_info->output_sub_size = output_sub_range; @@ -7703,12 +9701,14 @@ static void stbir__init_and_set_layout( STBIR_RESIZE * resize, stbir_pixel_layou resize->samplers = 0; resize->called_alloc = 0; resize->horizontal_filter = STBIR_FILTER_DEFAULT; - resize->horizontal_filter_kernel = 0; resize->horizontal_filter_support = 0; + resize->horizontal_filter_kernel = 0; resize->horizontal_filter_support = 0; resize->horizontal_filter_kernel_new = 0; resize->horizontal_filter_support_new = 0; resize->horizontal_filter_radius = 0; resize->vertical_filter = STBIR_FILTER_DEFAULT; - resize->vertical_filter_kernel = 0; resize->vertical_filter_support = 0; + resize->vertical_filter_kernel = 0; resize->vertical_filter_support = 0; resize->vertical_filter_kernel_new = 0; resize->vertical_filter_support_new = 0; resize->vertical_filter_radius = 0; resize->horizontal_edge = STBIR_EDGE_CLAMP; resize->vertical_edge = STBIR_EDGE_CLAMP; resize->input_s0 = 0; resize->input_t0 = 0; resize->input_s1 = 1; resize->input_t1 = 1; + resize->input_s_n0 = 0; resize->input_s_n1 = 1; resize->input_s_d = 1; + resize->input_t_n0 = 0; resize->input_t_n1 = 1; resize->input_t_d = 1; resize->output_subx = 0; resize->output_suby = 0; resize->output_subw = resize->output_w; resize->output_subh = resize->output_h; resize->input_data_type = data_type; resize->output_data_type = data_type; @@ -7718,8 +9718,8 @@ static void stbir__init_and_set_layout( STBIR_RESIZE * resize, stbir_pixel_layou } STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize, - const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero + const void *input_pixels, int input_w, int input_h, stbir_int64 input_stride_in_bytes, // stride can be zero + void *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, // stride can be zero stbir_pixel_layout pixel_layout, stbir_datatype data_type ) { resize->input_pixels = input_pixels; @@ -7763,7 +9763,7 @@ STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ) resize->samplers->user_data = user_data; } -STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes ) +STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, stbir_int64 input_stride_in_bytes, void * output_pixels, stbir_int64 output_stride_in_bytes ) { resize->input_pixels = input_pixels; resize->input_stride_in_bytes = input_stride_in_bytes; @@ -7790,10 +9790,38 @@ STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_f return 1; } +STBIRDEF int stbir_set_filter_callbacks_new( STBIR_RESIZE * resize, stbir__kernel_callback_new * horizontal_filter, stbir__support_callback_new * horizontal_support, stbir__radius_callback_new * horizontal_radius, stbir__kernel_callback_new * vertical_filter, stbir__support_callback_new * vertical_support, stbir__radius_callback_new * vertical_radius ) +{ + // a filter's extent is given one way: a support (filter pixels) or a radius (input pixels), not both + if ( ( horizontal_support && horizontal_radius ) || ( vertical_support && vertical_radius ) ) + return 0; + resize->horizontal_filter_kernel_new = horizontal_filter; + resize->horizontal_filter_support_new = horizontal_support; + resize->horizontal_filter_radius = horizontal_radius; + resize->horizontal_filter_kernel = 0; // (the old form's, if it was set before) + resize->horizontal_filter_support = 0; + resize->vertical_filter_kernel_new = vertical_filter; + resize->vertical_filter_support_new = vertical_support; + resize->vertical_filter_radius = vertical_radius; + resize->vertical_filter_kernel = 0; + resize->vertical_filter_support = 0; + resize->needs_rebuild = 1; + return 1; +} + +// the old form: the float kernels and supports are kept apart and the sampler calls them through a conversion STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support ) { - resize->horizontal_filter_kernel = horizontal_filter; resize->horizontal_filter_support = horizontal_support; - resize->vertical_filter_kernel = vertical_filter; resize->vertical_filter_support = vertical_support; + resize->horizontal_filter_kernel = horizontal_filter; + resize->horizontal_filter_support = horizontal_support; + resize->horizontal_filter_kernel_new = 0; + resize->horizontal_filter_support_new = 0; + resize->horizontal_filter_radius = 0; + resize->vertical_filter_kernel = vertical_filter; + resize->vertical_filter_support = vertical_support; + resize->vertical_filter_kernel_new = 0; + resize->vertical_filter_support_new = 0; + resize->vertical_filter_radius = 0; resize->needs_rebuild = 1; return 1; } @@ -7814,25 +9842,27 @@ STBIRDEF int stbir_set_non_pm_alpha_speed_over_quality( STBIR_RESIZE * resize, i return 1; } -STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ) // sets input region (full region by default) +STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, stbir_real s0, stbir_real t0, stbir_real s1, stbir_real t1 ) // sets input region (full region by default) { resize->input_s0 = s0; resize->input_t0 = t0; resize->input_s1 = s1; resize->input_t1 = t1; + resize->input_s_d = 0; // not in pixels: the fractions are worked out from the reals at build time + resize->input_t_d = 0; resize->needs_rebuild = 1; - // are we inbounds? - if ( ( s1 < stbir__small_float ) || ( (s1-s0) < stbir__small_float ) || - ( t1 < stbir__small_float ) || ( (t1-t0) < stbir__small_float ) || - ( s0 > (1.0f-stbir__small_float) ) || - ( t0 > (1.0f-stbir__small_float) ) ) + // are we inbounds? (a zero or negative area, or a region wholly past the image, is empty) + if ( ( s1 <= (stbir_real)0.0 ) || ( s1 <= s0 ) || + ( t1 <= (stbir_real)0.0 ) || ( t1 <= t0 ) || + ( s0 >= (stbir_real)1.0 ) || + ( t0 >= (stbir_real)1.0 ) ) return 0; return 1; } -STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets input region (full region by default) +STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets the output sub rect, in output pixels (the whole output by default) { resize->output_subx = subx; resize->output_suby = suby; @@ -7840,8 +9870,8 @@ STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, in resize->output_subh = subh; resize->needs_rebuild = 1; - // are we inbounds? - if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) ) + // is there anything in it? (a negative size is an empty rect - the resize then has nothing to do) + if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw <= 0 ) || ( subh <= 0 ) ) return 0; return 1; @@ -7849,25 +9879,31 @@ STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, in STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets both regions (full regions by default) { - double s0, t0, s1, t1; + stbir_real s0, t0, s1, t1; - s0 = ( (double)subx ) / ( (double)resize->output_w ); - t0 = ( (double)suby ) / ( (double)resize->output_h ); - s1 = ( (double)(subx+subw) ) / ( (double)resize->output_w ); - t1 = ( (double)(suby+subh) ) / ( (double)resize->output_h ); + s0 = ( (stbir_real)subx ) / ( (stbir_real)resize->output_w ); + t0 = ( (stbir_real)suby ) / ( (stbir_real)resize->output_h ); + s1 = ( (stbir_real)(subx+subw) ) / ( (stbir_real)resize->output_w ); + t1 = ( (stbir_real)(suby+subh) ) / ( (stbir_real)resize->output_h ); resize->input_s0 = s0; resize->input_t0 = t0; resize->input_s1 = s1; resize->input_t1 = t1; + resize->input_s_n0 = subx; + resize->input_s_n1 = subx + subw; + resize->input_s_d = resize->output_w; + resize->input_t_n0 = suby; + resize->input_t_n1 = suby + subh; + resize->input_t_d = resize->output_h; resize->output_subx = subx; resize->output_suby = suby; resize->output_subw = subw; resize->output_subh = subh; resize->needs_rebuild = 1; - // are we inbounds? - if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) ) + // is there anything in it? (a negative size is an empty rect - the resize then has nothing to do) + if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw <= 0 ) || ( subh <= 0 ) ) return 0; return 1; @@ -7901,35 +9937,32 @@ static int stbir__perform_build( STBIR_RESIZE * resize, int splits ) new_output_subx = resize->output_subx; new_output_suby = resize->output_suby; - // do horizontal clip and scale calcs - if ( !stbir__calculate_region_transform( &horizontal.scale_info, resize->output_w, &new_output_subx, resize->output_subw, resize->input_w, resize->input_s0, resize->input_s1 ) ) + // do horizontal clip and scale calcs (an empty region - zero or negative sizes, nothing in bounds - fails the build) + if ( !stbir__calculate_region_transform( &horizontal.scale_info, resize->output_w, &new_output_subx, resize->output_subw, resize->input_w, resize->input_s0, resize->input_s1, resize->input_s_n0, resize->input_s_n1, resize->input_s_d ) ) return 0; // do vertical clip and scale calcs - if ( !stbir__calculate_region_transform( &vertical.scale_info, resize->output_h, &new_output_suby, resize->output_subh, resize->input_h, resize->input_t0, resize->input_t1 ) ) + if ( !stbir__calculate_region_transform( &vertical.scale_info, resize->output_h, &new_output_suby, resize->output_subh, resize->input_h, resize->input_t0, resize->input_t1, resize->input_t_n0, resize->input_t_n1, resize->input_t_d ) ) return 0; // if nothing to do, just return if ( ( horizontal.scale_info.output_sub_size == 0 ) || ( vertical.scale_info.output_sub_size == 0 ) ) return 0; - stbir__set_sampler(&horizontal, resize->horizontal_filter, resize->horizontal_filter_kernel, resize->horizontal_filter_support, resize->horizontal_edge, &horizontal.scale_info, 1, resize->user_data ); + stbir__set_sampler(&horizontal, resize->horizontal_filter, resize->horizontal_filter_kernel_new, resize->horizontal_filter_support_new, resize->horizontal_filter_radius, resize->horizontal_filter_kernel, resize->horizontal_filter_support, resize->horizontal_edge, &horizontal.scale_info, 1, resize->user_data ); stbir__get_conservative_extents( &horizontal, &conservative, resize->user_data ); - stbir__set_sampler(&vertical, resize->vertical_filter, resize->vertical_filter_kernel, resize->vertical_filter_support, resize->vertical_edge, &vertical.scale_info, 0, resize->user_data ); + stbir__set_sampler(&vertical, resize->vertical_filter, resize->vertical_filter_kernel_new, resize->vertical_filter_support_new, resize->vertical_filter_radius, resize->vertical_filter_kernel, resize->vertical_filter_support, resize->vertical_edge, &vertical.scale_info, 0, resize->user_data ); - if ( ( vertical.scale_info.output_sub_size / splits ) < STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS ) // each split should be a minimum of 4 scanlines (handwavey choice) - { - splits = vertical.scale_info.output_sub_size / STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS; - if ( splits == 0 ) splits = 1; - } + // the split count can come down in the build (stbir__choose_split_grid): the caller gets the count it can use STBIR_PROFILE_BUILD_START( alloc ); - out_info = stbir__alloc_internal_mem_and_build_samplers( &horizontal, &vertical, &conservative, resize->input_pixel_layout_public, resize->output_pixel_layout_public, splits, new_output_subx, new_output_suby, resize->fast_alpha, resize->user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + out_info = stbir__alloc_internal_mem_and_build_samplers( &horizontal, &vertical, &conservative, resize->input_pixel_layout_public, resize->output_pixel_layout_public, splits, new_output_subx, new_output_suby, resize->fast_alpha, stbir__pixel_channels[ stbir__pixel_layout_convert_public_to_internal[ resize->input_pixel_layout_public ] ] * stbir__type_size[ resize->input_data_type ], resize->user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); STBIR_PROFILE_BUILD_END( alloc ); STBIR_PROFILE_BUILD_END( build ); if ( out_info ) { + splits = out_info->splits; resize->splits = splits; resize->samplers = out_info; resize->needs_rebuild = 0; @@ -7969,7 +10002,7 @@ STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int splits STBIR_PROFILE_BUILD_CLEAR( resize->samplers ); - return 1; + return resize->splits; // already built: the count that build settled on (a new request isn't applied without a rebuild) } STBIRDEF int stbir_build_samplers( STBIR_RESIZE * resize ) @@ -7992,15 +10025,9 @@ STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize ) } if ( !stbir_build_samplers( resize ) ) - return 0; + return 0; // (an empty resize fails the build too, so it comes out of here as a failure) resize->called_alloc = alloc_state; - - // if build_samplers succeeded (above), but there are no samplers set, then - // the area to stretch into was zero pixels, so don't do anything and return - // success - if ( resize->samplers == 0 ) - return 1; } else { @@ -8025,14 +10052,14 @@ STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start { STBIR_ASSERT( resize->samplers ); + // you **must** build samplers first when using split resize (and not change the resize after) + if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) ) + return 0; + // if we're just doing the whole thing, call full if ( ( split_start == -1 ) || ( ( split_start == 0 ) && ( split_count == resize->splits ) ) ) return stbir_resize_extended( resize ); - // you **must** build samplers first when using split resize - if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) ) - return 0; - if ( ( split_start >= resize->splits ) || ( split_start < 0 ) || ( ( split_start + split_count ) > resize->splits ) || ( split_count <= 0 ) ) return 0; @@ -8041,18 +10068,18 @@ STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start } -static void * stbir_quick_resize_helper( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +static void * stbir_quick_resize_helper( const void *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout, stbir_datatype data_type, stbir_edge edge, stbir_filter filter ) { STBIR_RESIZE resize; - int scanline_output_in_bytes; - int positive_output_stride_in_bytes; + stbir_int64 scanline_output_in_bytes; + stbir_int64 positive_output_stride_in_bytes; void * start_ptr; void * free_ptr; - scanline_output_in_bytes = output_w * stbir__type_size[ data_type ] * stbir__pixel_channels[ stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ]; - if ( scanline_output_in_bytes == 0 ) + scanline_output_in_bytes = (stbir_int64)output_w * stbir__type_size[ data_type ] * stbir__pixel_channels[ stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ]; + if ( scanline_output_in_bytes <= 0 ) return 0; // if zero stride, use scanline output @@ -8116,9 +10143,8 @@ static void * stbir_quick_resize_helper( const void *input_pixels , int input_w } - -STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout ) { return (unsigned char *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes, @@ -8126,8 +10152,8 @@ STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_p pixel_layout, STBIR_TYPE_UINT8, STBIR_EDGE_CLAMP, STBIR_FILTER_DEFAULT ); } -STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout ) { return (unsigned char *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes, @@ -8136,8 +10162,8 @@ STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pix } -STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout ) { return (float *) stbir_quick_resize_helper( input_pixels , input_w , input_h, input_stride_in_bytes, @@ -8146,8 +10172,8 @@ STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int inpu } -STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, +STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, stbir_int64 input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, stbir_int64 output_stride_in_bytes, stbir_pixel_layout pixel_layout, stbir_datatype data_type, stbir_edge edge, stbir_filter filter ) { @@ -8160,15 +10186,11 @@ STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input STBIRDEF void stbir_resize_build_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize ) { - static char const * bdescriptions[6] = { "Building", "Allocating", "Horizontal sampler", "Vertical sampler", "Coefficient cleanup", "Coefficient pivot" } ; + static char const * bdescriptions[6] = { "Building", "Allocating", "Horizontal sampler", "Vertical sampler", "Coefficient cleanup", "Scatter coefficients" } ; stbir__info* samp = resize->samplers; int i; - typedef int testa[ (STBIR__ARRAY_SIZE( bdescriptions ) == (STBIR__ARRAY_SIZE( samp->profile.array )-1) )?1:-1]; - typedef int testb[ (sizeof( samp->profile.array ) == (sizeof(samp->profile.named)) )?1:-1]; - typedef int testc[ (sizeof( info->clocks ) >= (sizeof(samp->profile.named)) )?1:-1]; - - for( i = 0 ; i < STBIR__ARRAY_SIZE( bdescriptions ) ; i++) + for( i = 0 ; i < (int)STBIR__ARRAY_SIZE( bdescriptions ) ; i++) info->clocks[i] = samp->profile.array[i+1]; info->total_clocks = samp->profile.named.total; @@ -8182,10 +10204,6 @@ STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * info, STBIR_ stbir__per_split_info * split_info; int s, i; - typedef int testa[ (STBIR__ARRAY_SIZE( descriptions ) == (STBIR__ARRAY_SIZE( split_info->profile.array )-1) )?1:-1]; - typedef int testb[ (sizeof( split_info->profile.array ) == (sizeof(split_info->profile.named)) )?1:-1]; - typedef int testc[ (sizeof( info->clocks ) >= (sizeof(split_info->profile.named)) )?1:-1]; - if ( split_start == -1 ) { split_start = 0; @@ -8203,7 +10221,7 @@ STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * info, STBIR_ split_info = resize->samplers->split_info + split_start; // sum up the profile from all the splits - for( i = 0 ; i < STBIR__ARRAY_SIZE( descriptions ) ; i++ ) + for( i = 0 ; i < (int)STBIR__ARRAY_SIZE( descriptions ) ; i++ ) { stbir_uint64 sum = 0; for( s = 0 ; s < split_count ; s++ ) @@ -8211,7 +10229,9 @@ STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * info, STBIR_ info->clocks[i] = sum; } - info->total_clocks = split_info->profile.named.total; + info->total_clocks = 0; + for( s = 0 ; s < split_count ; s++ ) + info->total_clocks += split_info[s].profile.named.total; info->descriptions = descriptions; info->count = STBIR__ARRAY_SIZE( descriptions ); } @@ -8269,6 +10289,29 @@ STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * info, STB #define stbir__decode_simdf4_flip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg) #define stbir__encode_simdf8_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( stbir__simdf8_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3),stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg) #define stbir__encode_simdf4_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg) +#define stbir__encode_simdf16_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf16_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg) // (one lane's order, the shuffle repeats it) +// u8 and u16 inputs are swizzled as bytes before the expansion when there is a byte shuffle +#ifdef STBIR_SIMD_BYTE_SHUFFLE +#define STBIR__SWZ8( p ) ( ( (p)*4 + stbir__decode_order0 ) | ( ( (p)*4 + stbir__decode_order1 ) << 8 ) | ( ( (p)*4 + stbir__decode_order2 ) << 16 ) | ( ( (p)*4 + stbir__decode_order3 ) << 24 ) ) +#define STBIR__SWZ16( p, a, b ) ( ( 2*((p)*4 + (a)) ) | ( ( 2*((p)*4 + (a)) + 1 ) << 8 ) | ( ( 2*((p)*4 + (b)) ) << 16 ) | ( ( 2*((p)*4 + (b)) + 1 ) << 24 ) ) +static STBIR__SIMDI_CONST4( STBIR__CODER_NAME( stbir__decode_bytes_swizzle ), STBIR__SWZ8(0), STBIR__SWZ8(1), STBIR__SWZ8(2), STBIR__SWZ8(3) ); +static STBIR__SIMDI_CONST4( STBIR__CODER_NAME( stbir__decode_words_swizzle ), STBIR__SWZ16(0,stbir__decode_order0,stbir__decode_order1), STBIR__SWZ16(0,stbir__decode_order2,stbir__decode_order3), STBIR__SWZ16(1,stbir__decode_order0,stbir__decode_order1), STBIR__SWZ16(1,stbir__decode_order2,stbir__decode_order3) ); +#define stbir__decode_simdi_flip_u8( ireg ) stbir__simdi_shuffle_bytes( ireg, ireg, STBIR__CONSTI( STBIR__CODER_NAME( stbir__decode_bytes_swizzle ) ) ) +#define stbir__decode_simdi_flip_u16( ireg ) stbir__simdi_shuffle_bytes( ireg, ireg, STBIR__CONSTI( STBIR__CODER_NAME( stbir__decode_words_swizzle ) ) ) +#define stbir__decode_simdf8_flip_i( reg ) +#define stbir__decode_simdf4_flip_i( reg ) +#define stbir__decode_simdf16_flip_i( reg ) +#ifdef STBIR_SIMD16 +#define stbir__decode_simdi8_flip_u16( ireg ) stbir__simdi8_shuffle_bytes( ireg, ireg, STBIR__CONSTI( STBIR__CODER_NAME( stbir__decode_words_swizzle ) ) ) // (the 16-wide u16 decoder loads 32 bytes: the same table in both lanes) +#endif +#else +#define stbir__decode_simdi_flip_u8( ireg ) +#define stbir__decode_simdi_flip_u16( ireg ) +#define stbir__decode_simdi8_flip_u16( ireg ) +#define stbir__decode_simdf8_flip_i( reg ) stbir__decode_simdf8_flip( reg ) +#define stbir__decode_simdf4_flip_i( reg ) stbir__decode_simdf4_flip( reg ) +#define stbir__decode_simdf16_flip_i( reg ) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf16_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg) +#endif #else #define stbir__decode_order0 0 #define stbir__decode_order1 1 @@ -8280,17 +10323,26 @@ STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * info, STB #define stbir__encode_order3 3 #define stbir__decode_simdf8_flip(reg) #define stbir__decode_simdf4_flip(reg) +#define stbir__decode_simdi_flip_u8( ireg ) +#define stbir__decode_simdi_flip_u16( ireg ) +#define stbir__decode_simdi8_flip_u16( ireg ) +#define stbir__decode_simdf8_flip_i( reg ) +#define stbir__decode_simdf4_flip_i( reg ) +#define stbir__decode_simdf16_flip_i( reg ) #define stbir__encode_simdf8_unflip(reg) #define stbir__encode_simdf4_unflip(reg) +#define stbir__encode_simdf16_unflip(reg) #endif -#ifdef STBIR_SIMD8 +#if defined(STBIR_SIMD16) +#define stbir__encode_simdfX_unflip stbir__encode_simdf16_unflip +#elif defined(STBIR_SIMD8) #define stbir__encode_simdfX_unflip stbir__encode_simdf8_unflip #else #define stbir__encode_simdfX_unflip stbir__encode_simdf4_unflip #endif -static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -8304,18 +10356,30 @@ static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * d STBIR_NO_UNROLL_LOOP_START_INF_FOR for(;;) { - #ifdef STBIR_SIMD8 + #if defined(STBIR_SIMD16) + stbir__simdi i; stbir__simdi16 o; + stbir__simdf16 of; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); + stbir__simdi16_expand_u8_to_u32( o, i ); + stbir__simdi16_convert_i32_to_float( of, o ); + stbir__simdf16_mult( of, of, STBIR_max_uint8_as_float_inverted16 ); + stbir__decode_simdf16_flip_i( of ); + stbir__simdf16_store( decode, of ); + #elif defined(STBIR_SIMD8) stbir__simdi i; stbir__simdi8 o0,o1; stbir__simdf8 of0, of1; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); stbir__simdi8_expand_u8_to_u32( o0, o1, i ); stbir__simdi8_convert_i32_to_float( of0, o0 ); stbir__simdi8_convert_i32_to_float( of1, o1 ); stbir__simdf8_mult( of0, of0, STBIR_max_uint8_as_float_inverted8); stbir__simdf8_mult( of1, of1, STBIR_max_uint8_as_float_inverted8); - stbir__decode_simdf8_flip( of0 ); - stbir__decode_simdf8_flip( of1 ); + stbir__decode_simdf8_flip_i( of0 ); + stbir__decode_simdf8_flip_i( of1 ); stbir__simdf8_store( decode + 0, of0 ); stbir__simdf8_store( decode + 8, of1 ); #else @@ -8323,6 +10387,7 @@ static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * d stbir__simdf of0, of1, of2, of3; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i); stbir__simdi_convert_i32_to_float( of0, o0 ); stbir__simdi_convert_i32_to_float( of1, o1 ); @@ -8332,10 +10397,10 @@ static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * d stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); stbir__simdf_mult( of2, of2, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); stbir__simdf_mult( of3, of3, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); - stbir__decode_simdf4_flip( of0 ); - stbir__decode_simdf4_flip( of1 ); - stbir__decode_simdf4_flip( of2 ); - stbir__decode_simdf4_flip( of3 ); + stbir__decode_simdf4_flip_i( of0 ); + stbir__decode_simdf4_flip_i( of1 ); + stbir__decode_simdf4_flip_i( of2 ); + stbir__decode_simdf4_flip_i( of3 ); stbir__simdf_store( decode + 0, of0 ); stbir__simdf_store( decode + 4, of1 ); stbir__simdf_store( decode + 8, of2 ); @@ -8392,43 +10457,44 @@ static float * STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * d return decode_end; } -static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp; unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= stbir__simdfX_float_count*2 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= stbir__simdfX_float_count*2 ) { - float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; - end_output -= stbir__simdfX_float_count*2; - STBIR_NO_UNROLL_LOOP_START_INF_FOR - for(;;) + stbir__simdfX e0, e1; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode ); + stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + #if defined(STBIR_SIMD16) + { + stbir__simdi8 i; + stbir__simdf16_pack_to_32bytes( i, e0, e1 ); + stbir__simdi8_store( output, i ); + } + #elif defined(STBIR_SIMD8) { - stbir__simdfX e0, e1; stbir__simdi i; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode ); - stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode+stbir__simdfX_float_count ); - stbir__encode_simdfX_unflip( e0 ); - stbir__encode_simdfX_unflip( e1 ); - #ifdef STBIR_SIMD8 stbir__simdf8_pack_to_16bytes( i, e0, e1 ); stbir__simdi_store( output, i ); - #else + } + #else + { + stbir__simdi i; stbir__simdf_pack_to_8bytes( i, e0, e1 ); stbir__simdi_store2( output, i ); - #endif - encode += stbir__simdfX_float_count*2; - output += stbir__simdfX_float_count*2; - if ( output <= end_output ) - continue; - if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; } - return; + #endif + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; } // try to do blocks of 4 when you can @@ -8456,14 +10522,29 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { + // read the whole pixel before writing any of it - the encode can be in place, and a swizzle reads a channel after + // the one it lands on has been overwritten stbir__simdf e0; - STBIR_NO_UNROLL(encode); - stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_uint8( e0 ); #if stbir__coder_min_num >= 2 - stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_uint8( e0 ); + stbir__simdf e1; #endif #if stbir__coder_min_num >= 3 - stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_uint8( e0 ); + stbir__simdf e2; + #endif + STBIR_NO_UNROLL(encode); + stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order0 ); + #if stbir__coder_min_num >= 2 + stbir__simdf_madd1_mem( e1, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order1 ); + #endif + #if stbir__coder_min_num >= 3 + stbir__simdf_madd1_mem( e2, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order2 ); + #endif + output[0] = stbir__simdf_convert_float_to_uint8( e0 ); + #if stbir__coder_min_num >= 2 + output[1] = stbir__simdf_convert_float_to_uint8( e1 ); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__simdf_convert_float_to_uint8( e2 ); #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -8477,11 +10558,18 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu output += 4; while( output <= end_output ) { - float f; - f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f; - f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f; - f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f; - f = encode[stbir__encode_order3] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f; + float f0 = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; + float f1 = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; + float f2 = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; + float f3 = encode[stbir__encode_order3] * stbir__max_uint8_as_float + 0.5f; + STBIR_CLAMP(f0, 0, 255); + STBIR_CLAMP(f1, 0, 255); + STBIR_CLAMP(f2, 0, 255); + STBIR_CLAMP(f3, 0, 255); + output[0-4] = (unsigned char)f0; + output[1-4] = (unsigned char)f1; + output[2-4] = (unsigned char)f2; + output[3-4] = (unsigned char)f3; output += 4; encode += 4; } @@ -8493,14 +10581,23 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - float f; - STBIR_NO_UNROLL(encode); - f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f; + float f0 = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; #if stbir__coder_min_num >= 2 - f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f; + float f1 = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; #endif #if stbir__coder_min_num >= 3 - f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f; + float f2 = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; + #endif + STBIR_NO_UNROLL(encode); + STBIR_CLAMP(f0, 0, 255); + output[0] = (unsigned char)f0; + #if stbir__coder_min_num >= 2 + STBIR_CLAMP(f1, 0, 255); + output[1] = (unsigned char)f1; + #endif + #if stbir__coder_min_num >= 3 + STBIR_CLAMP(f2, 0, 255); + output[2] = (unsigned char)f2; #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -8509,7 +10606,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outpu #endif } -static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -8523,16 +10620,27 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, i STBIR_NO_UNROLL_LOOP_START_INF_FOR for(;;) { - #ifdef STBIR_SIMD8 + #if defined(STBIR_SIMD16) + stbir__simdi i; stbir__simdi16 o; + stbir__simdf16 of; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); + stbir__simdi16_expand_u8_to_u32( o, i ); + stbir__simdi16_convert_i32_to_float( of, o ); + stbir__decode_simdf16_flip_i( of ); + stbir__simdf16_store( decode, of ); + #elif defined(STBIR_SIMD8) stbir__simdi i; stbir__simdi8 o0,o1; stbir__simdf8 of0, of1; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); stbir__simdi8_expand_u8_to_u32( o0, o1, i ); stbir__simdi8_convert_i32_to_float( of0, o0 ); stbir__simdi8_convert_i32_to_float( of1, o1 ); - stbir__decode_simdf8_flip( of0 ); - stbir__decode_simdf8_flip( of1 ); + stbir__decode_simdf8_flip_i( of0 ); + stbir__decode_simdf8_flip_i( of1 ); stbir__simdf8_store( decode + 0, of0 ); stbir__simdf8_store( decode + 8, of1 ); #else @@ -8540,15 +10648,16 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, i stbir__simdf of0, of1, of2, of3; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u8( i ); stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i); stbir__simdi_convert_i32_to_float( of0, o0 ); stbir__simdi_convert_i32_to_float( of1, o1 ); stbir__simdi_convert_i32_to_float( of2, o2 ); stbir__simdi_convert_i32_to_float( of3, o3 ); - stbir__decode_simdf4_flip( of0 ); - stbir__decode_simdf4_flip( of1 ); - stbir__decode_simdf4_flip( of2 ); - stbir__decode_simdf4_flip( of3 ); + stbir__decode_simdf4_flip_i( of0 ); + stbir__decode_simdf4_flip_i( of1 ); + stbir__decode_simdf4_flip_i( of2 ); + stbir__decode_simdf4_flip_i( of3 ); stbir__simdf_store( decode + 0, of0 ); stbir__simdf_store( decode + 4, of1 ); stbir__simdf_store( decode + 8, of2 ); @@ -8604,43 +10713,44 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, i return decode_end; } -static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp; unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= stbir__simdfX_float_count*2 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= stbir__simdfX_float_count*2 ) { - float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; - end_output -= stbir__simdfX_float_count*2; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) + stbir__simdfX e0, e1; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); + stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + #if defined(STBIR_SIMD16) + { + stbir__simdi8 i; + stbir__simdf16_pack_to_32bytes( i, e0, e1 ); + stbir__simdi8_store( output, i ); + } + #elif defined(STBIR_SIMD8) { - stbir__simdfX e0, e1; stbir__simdi i; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); - stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); - stbir__encode_simdfX_unflip( e0 ); - stbir__encode_simdfX_unflip( e1 ); - #ifdef STBIR_SIMD8 stbir__simdf8_pack_to_16bytes( i, e0, e1 ); stbir__simdi_store( output, i ); - #else + } + #else + { + stbir__simdi i; stbir__simdf_pack_to_8bytes( i, e0, e1 ); stbir__simdi_store2( output, i ); - #endif - encode += stbir__simdfX_float_count*2; - output += stbir__simdfX_float_count*2; - if ( output <= end_output ) - continue; - if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; } - return; + #endif + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; } // try to do blocks of 4 when you can @@ -8670,11 +10780,18 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int output += 4; while( output <= end_output ) { - float f; - f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f; - f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f; - f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f; - f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f; + float f0 = encode[stbir__encode_order0] + 0.5f; + float f1 = encode[stbir__encode_order1] + 0.5f; + float f2 = encode[stbir__encode_order2] + 0.5f; + float f3 = encode[stbir__encode_order3] + 0.5f; + STBIR_CLAMP(f0, 0, 255); + STBIR_CLAMP(f1, 0, 255); + STBIR_CLAMP(f2, 0, 255); + STBIR_CLAMP(f3, 0, 255); + output[0-4] = (unsigned char)f0; + output[1-4] = (unsigned char)f1; + output[2-4] = (unsigned char)f2; + output[3-4] = (unsigned char)f3; output += 4; encode += 4; } @@ -8688,14 +10805,23 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - float f; - STBIR_NO_UNROLL(encode); - f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f; + float f0 = encode[stbir__encode_order0] + 0.5f; #if stbir__coder_min_num >= 2 - f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f; + float f1 = encode[stbir__encode_order1] + 0.5f; #endif #if stbir__coder_min_num >= 3 - f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f; + float f2 = encode[stbir__encode_order2] + 0.5f; + #endif + STBIR_NO_UNROLL(encode); + STBIR_CLAMP(f0, 0, 255); + output[0] = (unsigned char)f0; + #if stbir__coder_min_num >= 2 + STBIR_CLAMP(f1, 0, 255); + output[1] = (unsigned char)f1; + #endif + #if stbir__coder_min_num >= 3 + STBIR_CLAMP(f2, 0, 255); + output[2] = (unsigned char)f2; #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -8703,7 +10829,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int #endif } -static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -8745,15 +10871,19 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int } #define stbir__min_max_shift20( i, f ) \ + do { \ stbir__simdf_max( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_zero )) ); \ stbir__simdf_min( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_one )) ); \ - stbir__simdi_32shr( i, stbir_simdi_castf( f ), 20 ); + stbir__simdi_32shr( i, stbir_simdi_castf( f ), 20 ); \ + } while(0) #define stbir__scale_and_convert( i, f ) \ + do { \ stbir__simdf_madd( f, STBIR__CONSTF( STBIR_simd_point5 ), STBIR__CONSTF( STBIR_max_uint8_as_float ), f ); \ stbir__simdf_max( f, f, stbir__simdf_zeroP() ); \ stbir__simdf_min( f, f, STBIR__CONSTF( STBIR_max_uint8_as_float ) ); \ - stbir__simdf_convert_float_to_i32( i, f ); + stbir__simdf_convert_float_to_i32( i, f ); \ + } while(0) #define stbir__linear_to_srgb_finish( i, f ) \ { \ @@ -8765,92 +10895,75 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int stbir__simdi_32shr( i, i, 16 ); \ } -#define stbir__simdi_table_lookup2( v0,v1, table ) \ -{ \ - stbir__simdi_u32 temp0,temp1; \ - temp0.m128i_i128 = v0; \ - temp1.m128i_i128 = v1; \ - temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ - temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ - v0 = temp0.m128i_i128; \ - v1 = temp1.m128i_i128; \ -} +// look up sRGB bytes for 4 float lanes per register (indexed by the float bits shifted right 20, see fp32_to_srgb8_tab4) +#define stbir__simdi_srgb_lookup_one( v, temp ) \ + do { \ + temp.m128i_i128 = v; \ + temp.m128i_u32[0] = STBIR__SRGB8_TAB4( temp.m128i_i32[0] ); \ + temp.m128i_u32[1] = STBIR__SRGB8_TAB4( temp.m128i_i32[1] ); \ + temp.m128i_u32[2] = STBIR__SRGB8_TAB4( temp.m128i_i32[2] ); \ + temp.m128i_u32[3] = STBIR__SRGB8_TAB4( temp.m128i_i32[3] ); \ + v = temp.m128i_i128; \ + } while(0) -#define stbir__simdi_table_lookup3( v0,v1,v2, table ) \ -{ \ - stbir__simdi_u32 temp0,temp1,temp2; \ - temp0.m128i_i128 = v0; \ - temp1.m128i_i128 = v1; \ - temp2.m128i_i128 = v2; \ - temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ - temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ - temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \ - v0 = temp0.m128i_i128; \ - v1 = temp1.m128i_i128; \ - v2 = temp2.m128i_i128; \ -} +#define stbir__simdi_srgb_lookup2( v0,v1 ) \ + do { \ + stbir__simdi_u32 temp0,temp1; \ + stbir__simdi_srgb_lookup_one( v0, temp0 ); \ + stbir__simdi_srgb_lookup_one( v1, temp1 ); \ + } while(0) -#define stbir__simdi_table_lookup4( v0,v1,v2,v3, table ) \ -{ \ - stbir__simdi_u32 temp0,temp1,temp2,temp3; \ - temp0.m128i_i128 = v0; \ - temp1.m128i_i128 = v1; \ - temp2.m128i_i128 = v2; \ - temp3.m128i_i128 = v3; \ - temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ - temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ - temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \ - temp3.m128i_u32[0] = table[temp3.m128i_i32[0]]; temp3.m128i_u32[1] = table[temp3.m128i_i32[1]]; temp3.m128i_u32[2] = table[temp3.m128i_i32[2]]; temp3.m128i_u32[3] = table[temp3.m128i_i32[3]]; \ - v0 = temp0.m128i_i128; \ - v1 = temp1.m128i_i128; \ - v2 = temp2.m128i_i128; \ - v3 = temp3.m128i_i128; \ -} +#define stbir__simdi_srgb_lookup3( v0,v1,v2 ) \ + do { \ + stbir__simdi_u32 temp0,temp1,temp2; \ + stbir__simdi_srgb_lookup_one( v0, temp0 ); \ + stbir__simdi_srgb_lookup_one( v1, temp1 ); \ + stbir__simdi_srgb_lookup_one( v2, temp2 ); \ + } while(0) -static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int width_times_channels, float const * encode ) +#define stbir__simdi_srgb_lookup4( v0,v1,v2,v3 ) \ + do { \ + stbir__simdi_u32 temp0,temp1,temp2,temp3; \ + stbir__simdi_srgb_lookup_one( v0, temp0 ); \ + stbir__simdi_srgb_lookup_one( v1, temp1 ); \ + stbir__simdi_srgb_lookup_one( v2, temp2 ); \ + stbir__simdi_srgb_lookup_one( v3, temp3 ); \ + } while(0) + +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= 16 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= 16 ) { - float const * end_encode_m16 = encode + width_times_channels - 16; - end_output -= 16; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdf f0, f1, f2, f3; - stbir__simdi i0, i1, i2, i3; - STBIR_SIMD_NO_UNROLL(encode); + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; + STBIR_SIMD_NO_UNROLL(encode); - stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); - stbir__min_max_shift20( i0, f0 ); - stbir__min_max_shift20( i1, f1 ); - stbir__min_max_shift20( i2, f2 ); - stbir__min_max_shift20( i3, f3 ); + stbir__min_max_shift20( i0, f0 ); + stbir__min_max_shift20( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__min_max_shift20( i3, f3 ); - stbir__simdi_table_lookup4( i0, i1, i2, i3, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + stbir__simdi_srgb_lookup4( i0, i1, i2, i3 ); - stbir__linear_to_srgb_finish( i0, f0 ); - stbir__linear_to_srgb_finish( i1, f1 ); - stbir__linear_to_srgb_finish( i2, f2 ); - stbir__linear_to_srgb_finish( i3, f3 ); + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i1, f1 ); + stbir__linear_to_srgb_finish( i2, f2 ); + stbir__linear_to_srgb_finish( i3, f3 ); - stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); - encode += 16; - output += 16; - if ( output <= end_output ) - continue; - if ( output == ( end_output + 16 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m16; - } - return; + encode += 16; + output += 16; } #endif @@ -8860,12 +10973,17 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w STBIR_SIMD_NO_UNROLL_LOOP_START while ( output <= end_output ) { + float e0, e1, e2, e3; STBIR_SIMD_NO_UNROLL(encode); - output[0-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] ); - output[1-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] ); - output[2-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] ); - output[3-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order3] ); + e0 = encode[stbir__encode_order0]; + e1 = encode[stbir__encode_order1]; + e2 = encode[stbir__encode_order2]; + e3 = encode[stbir__encode_order3]; + output[0-4] = stbir__linear_to_srgb_uchar( e0 ); + output[1-4] = stbir__linear_to_srgb_uchar( e1 ); + output[2-4] = stbir__linear_to_srgb_uchar( e2 ); + output[3-4] = stbir__linear_to_srgb_uchar( e3 ); output += 4; encode += 4; @@ -8878,13 +10996,27 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - STBIR_NO_UNROLL(encode); - output[0] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] ); + float e0; #if stbir__coder_min_num >= 2 - output[1] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] ); + float e1; #endif #if stbir__coder_min_num >= 3 - output[2] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] ); + float e2; + #endif + STBIR_NO_UNROLL(encode); + e0 = encode[stbir__encode_order0]; + #if stbir__coder_min_num >= 2 + e1 = encode[stbir__encode_order1]; + #endif + #if stbir__coder_min_num >= 3 + e2 = encode[stbir__encode_order2]; + #endif + output[0] = stbir__linear_to_srgb_uchar( e0 ); + #if stbir__coder_min_num >= 2 + output[1] = stbir__linear_to_srgb_uchar( e1 ); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__linear_to_srgb_uchar( e2 ); #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -8894,7 +11026,7 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int w #if ( stbir__coder_min_num == 4 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) ) -static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb4_linearalpha)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb4_linearalpha)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -8912,55 +11044,47 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb4_linearalpha)( float * } -static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= 16 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= 16 ) { - float const * end_encode_m16 = encode + width_times_channels - 16; - end_output -= 16; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdf f0, f1, f2, f3; - stbir__simdi i0, i1, i2, i3; + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); - stbir__min_max_shift20( i0, f0 ); - stbir__min_max_shift20( i1, f1 ); - stbir__min_max_shift20( i2, f2 ); - stbir__scale_and_convert( i3, f3 ); + stbir__min_max_shift20( i0, f0 ); + stbir__min_max_shift20( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__scale_and_convert( i3, f3 ); - stbir__simdi_table_lookup3( i0, i1, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + stbir__simdi_srgb_lookup3( i0, i1, i2 ); - stbir__linear_to_srgb_finish( i0, f0 ); - stbir__linear_to_srgb_finish( i1, f1 ); - stbir__linear_to_srgb_finish( i2, f2 ); + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i1, f1 ); + stbir__linear_to_srgb_finish( i2, f2 ); - stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); - output += 16; - encode += 16; + output += 16; + encode += 16; - if ( output <= end_output ) - continue; - if ( output == ( end_output + 16 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m16; - } - return; } #endif + // the SIMD loop above can have finished the row, so this has to test first STBIR_SIMD_NO_UNROLL_LOOP_START - do { + while ( output < end_output ) + { float f; STBIR_SIMD_NO_UNROLL(encode); @@ -8974,14 +11098,14 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * o output += 4; encode += 4; - } while( output < end_output ); + } } #endif #if ( stbir__coder_min_num == 2 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) ) -static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb2_linearalpha)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb2_linearalpha)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -9006,53 +11130,45 @@ static float * STBIR__CODER_NAME(stbir__decode_uint8_srgb2_linearalpha)( float * return decode_end; } -static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= 16 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= 16 ) { - float const * end_encode_m16 = encode + width_times_channels - 16; - end_output -= 16; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdf f0, f1, f2, f3; - stbir__simdi i0, i1, i2, i3; + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); - stbir__min_max_shift20( i0, f0 ); - stbir__scale_and_convert( i1, f1 ); - stbir__min_max_shift20( i2, f2 ); - stbir__scale_and_convert( i3, f3 ); + stbir__min_max_shift20( i0, f0 ); + stbir__scale_and_convert( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__scale_and_convert( i3, f3 ); - stbir__simdi_table_lookup2( i0, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + stbir__simdi_srgb_lookup2( i0, i2 ); - stbir__linear_to_srgb_finish( i0, f0 ); - stbir__linear_to_srgb_finish( i2, f2 ); + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i2, f2 ); - stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); - output += 16; - encode += 16; - if ( output <= end_output ) - continue; - if ( output == ( end_output + 16 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m16; - } - return; + output += 16; + encode += 16; } #endif + // the SIMD loop above can have finished the row, so this has to test first STBIR_SIMD_NO_UNROLL_LOOP_START - do { + while ( output < end_output ) + { float f; STBIR_SIMD_NO_UNROLL(encode); @@ -9064,60 +11180,74 @@ static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * o output += 2; encode += 2; - } while( output < end_output ); + } } #endif -static float * STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; unsigned short const * input = (unsigned short const *)inputp; #ifdef STBIR_SIMD - unsigned short const * end_input_m8 = input + width_times_channels - 8; - if ( width_times_channels >= 8 ) + // a block is 16 words at 16 wide (one 32 byte load), 8 otherwise + unsigned short const * end_input_mb = input + width_times_channels - STBIR__U16_DECODE_BLOCK; + if ( width_times_channels >= STBIR__U16_DECODE_BLOCK ) { - decode_end -= 8; + decode_end -= STBIR__U16_DECODE_BLOCK; STBIR_NO_UNROLL_LOOP_START_INF_FOR for(;;) { - #ifdef STBIR_SIMD8 + #if defined(STBIR_SIMD16) + stbir__simdi8 i; stbir__simdi16 o; + stbir__simdf16 of; + STBIR_NO_UNROLL(decode); + stbir__simdi8_load( i, input ); + stbir__decode_simdi8_flip_u16( i ); + stbir__simdi16_expand_u16_to_u32( o, i ); + stbir__simdi16_convert_i32_to_float( of, o ); + stbir__simdf16_mult( of, of, STBIR_max_uint16_as_float_inverted16 ); + stbir__decode_simdf16_flip_i( of ); + stbir__simdf16_store( decode, of ); + #elif defined(STBIR_SIMD8) stbir__simdi i; stbir__simdi8 o; stbir__simdf8 of; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u16( i ); stbir__simdi8_expand_u16_to_u32( o, i ); stbir__simdi8_convert_i32_to_float( of, o ); stbir__simdf8_mult( of, of, STBIR_max_uint16_as_float_inverted8); - stbir__decode_simdf8_flip( of ); + stbir__decode_simdf8_flip_i( of ); stbir__simdf8_store( decode + 0, of ); #else stbir__simdi i, o0, o1; stbir__simdf of0, of1; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u16( i ); stbir__simdi_expand_u16_to_u32( o0,o1,i ); stbir__simdi_convert_i32_to_float( of0, o0 ); stbir__simdi_convert_i32_to_float( of1, o1 ); stbir__simdf_mult( of0, of0, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted) ); stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted)); - stbir__decode_simdf4_flip( of0 ); - stbir__decode_simdf4_flip( of1 ); + stbir__decode_simdf4_flip_i( of0 ); + stbir__decode_simdf4_flip_i( of1 ); stbir__simdf_store( decode + 0, of0 ); stbir__simdf_store( decode + 4, of1 ); #endif - decode += 8; - input += 8; + decode += STBIR__U16_DECODE_BLOCK; + input += STBIR__U16_DECODE_BLOCK; if ( decode <= decode_end ) continue; - if ( decode == ( decode_end + 8 ) ) + if ( decode == ( decode_end + STBIR__U16_DECODE_BLOCK ) ) break; decode = decode_end; // backup and do last couple - input = end_input_m8; + input = end_input_mb; } - return decode_end + 8; + return decode_end + STBIR__U16_DECODE_BLOCK; } #endif @@ -9159,39 +11289,29 @@ static float * STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * de } -static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp; unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels; #ifdef STBIR_SIMD { - if ( width_times_channels >= stbir__simdfX_float_count*2 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= stbir__simdfX_float_count*2 ) { - float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; - end_output -= stbir__simdfX_float_count*2; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdfX e0, e1; - stbir__simdiX i; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode ); - stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode+stbir__simdfX_float_count ); - stbir__encode_simdfX_unflip( e0 ); - stbir__encode_simdfX_unflip( e1 ); - stbir__simdfX_pack_to_words( i, e0, e1 ); - stbir__simdiX_store( output, i ); - encode += stbir__simdfX_float_count*2; - output += stbir__simdfX_float_count*2; - if ( output <= end_output ) - continue; - if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; - } - return; + stbir__simdfX e0, e1; + stbir__simdiX i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode ); + stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_pack_to_words( i, e0, e1 ); + stbir__simdiX_store( output, i ); + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; } } @@ -9220,14 +11340,29 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - stbir__simdf e; - STBIR_NO_UNROLL(encode); - stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_short( e ); + // read the whole pixel before writing any of it - the encode can be in place, and a swizzle reads a channel after + // the one it lands on has been overwritten + stbir__simdf e0; #if stbir__coder_min_num >= 2 - stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_short( e ); + stbir__simdf e1; #endif #if stbir__coder_min_num >= 3 - stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_short( e ); + stbir__simdf e2; + #endif + STBIR_NO_UNROLL(encode); + stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order0 ); + #if stbir__coder_min_num >= 2 + stbir__simdf_madd1_mem( e1, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order1 ); + #endif + #if stbir__coder_min_num >= 3 + stbir__simdf_madd1_mem( e2, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order2 ); + #endif + output[0] = stbir__simdf_convert_float_to_short( e0 ); + #if stbir__coder_min_num >= 2 + output[1] = stbir__simdf_convert_float_to_short( e1 ); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__simdf_convert_float_to_short( e2 ); #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -9242,12 +11377,17 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { - float f; + // read the whole pixel before writing any of it - the encode can be in place, and a swizzle reads a channel after + // the one it lands on has been overwritten + float f0 = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; + float f1 = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; + float f2 = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; + float f3 = encode[stbir__encode_order3] * stbir__max_uint16_as_float + 0.5f; STBIR_SIMD_NO_UNROLL(encode); - f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f; - f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f; - f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f; - f = encode[stbir__encode_order3] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f; + STBIR_CLAMP(f0, 0, 65535); output[0-4] = (unsigned short)f0; + STBIR_CLAMP(f1, 0, 65535); output[1-4] = (unsigned short)f1; + STBIR_CLAMP(f2, 0, 65535); output[2-4] = (unsigned short)f2; + STBIR_CLAMP(f3, 0, 65535); output[3-4] = (unsigned short)f3; output += 4; encode += 4; } @@ -9259,14 +11399,20 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - float f; - STBIR_NO_UNROLL(encode); - f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f; + float f0 = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; #if stbir__coder_min_num >= 2 - f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f; + float f1 = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; #endif #if stbir__coder_min_num >= 3 - f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f; + float f2 = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; + #endif + STBIR_NO_UNROLL(encode); + STBIR_CLAMP(f0, 0, 65535); output[0] = (unsigned short)f0; + #if stbir__coder_min_num >= 2 + STBIR_CLAMP(f1, 0, 65535); output[1] = (unsigned short)f1; + #endif + #if stbir__coder_min_num >= 3 + STBIR_CLAMP(f2, 0, 65535); output[2] = (unsigned short)f2; #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -9275,52 +11421,64 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * output #endif } -static float * STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; unsigned short const * input = (unsigned short const *)inputp; #ifdef STBIR_SIMD - unsigned short const * end_input_m8 = input + width_times_channels - 8; - if ( width_times_channels >= 8 ) + unsigned short const * end_input_mb = input + width_times_channels - STBIR__U16_DECODE_BLOCK; + if ( width_times_channels >= STBIR__U16_DECODE_BLOCK ) { - decode_end -= 8; + decode_end -= STBIR__U16_DECODE_BLOCK; STBIR_NO_UNROLL_LOOP_START_INF_FOR for(;;) { - #ifdef STBIR_SIMD8 + #if defined(STBIR_SIMD16) + stbir__simdi8 i; stbir__simdi16 o; + stbir__simdf16 of; + STBIR_NO_UNROLL(decode); + stbir__simdi8_load( i, input ); + stbir__decode_simdi8_flip_u16( i ); + stbir__simdi16_expand_u16_to_u32( o, i ); + stbir__simdi16_convert_i32_to_float( of, o ); + stbir__decode_simdf16_flip_i( of ); + stbir__simdf16_store( decode, of ); + #elif defined(STBIR_SIMD8) stbir__simdi i; stbir__simdi8 o; stbir__simdf8 of; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u16( i ); stbir__simdi8_expand_u16_to_u32( o, i ); stbir__simdi8_convert_i32_to_float( of, o ); - stbir__decode_simdf8_flip( of ); + stbir__decode_simdf8_flip_i( of ); stbir__simdf8_store( decode + 0, of ); #else stbir__simdi i, o0, o1; stbir__simdf of0, of1; STBIR_NO_UNROLL(decode); stbir__simdi_load( i, input ); + stbir__decode_simdi_flip_u16( i ); stbir__simdi_expand_u16_to_u32( o0, o1, i ); stbir__simdi_convert_i32_to_float( of0, o0 ); stbir__simdi_convert_i32_to_float( of1, o1 ); - stbir__decode_simdf4_flip( of0 ); - stbir__decode_simdf4_flip( of1 ); + stbir__decode_simdf4_flip_i( of0 ); + stbir__decode_simdf4_flip_i( of1 ); stbir__simdf_store( decode + 0, of0 ); stbir__simdf_store( decode + 4, of1 ); #endif - decode += 8; - input += 8; + decode += STBIR__U16_DECODE_BLOCK; + input += STBIR__U16_DECODE_BLOCK; if ( decode <= decode_end ) continue; - if ( decode == ( decode_end + 8 ) ) + if ( decode == ( decode_end + STBIR__U16_DECODE_BLOCK ) ) break; decode = decode_end; // backup and do last couple - input = end_input_m8; + input = end_input_mb; } - return decode_end + 8; + return decode_end + STBIR__U16_DECODE_BLOCK; } #endif @@ -9361,39 +11519,29 @@ static float * STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, return decode_end; } -static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp; unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels; #ifdef STBIR_SIMD { - if ( width_times_channels >= stbir__simdfX_float_count*2 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= stbir__simdfX_float_count*2 ) { - float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; - end_output -= stbir__simdfX_float_count*2; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdfX e0, e1; - stbir__simdiX i; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); - stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); - stbir__encode_simdfX_unflip( e0 ); - stbir__encode_simdfX_unflip( e1 ); - stbir__simdfX_pack_to_words( i, e0, e1 ); - stbir__simdiX_store( output, i ); - encode += stbir__simdfX_float_count*2; - output += stbir__simdfX_float_count*2; - if ( output <= end_output ) - continue; - if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; - } - return; + stbir__simdfX e0, e1; + stbir__simdiX i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); + stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_pack_to_words( i, e0, e1 ); + stbir__simdiX_store( output, i ); + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; } } @@ -9425,12 +11573,17 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { - float f; + // read the whole pixel before writing any of it - the encode can be in place, and a swizzle reads a channel after + // the one it lands on has been overwritten + float f0 = encode[stbir__encode_order0] + 0.5f; + float f1 = encode[stbir__encode_order1] + 0.5f; + float f2 = encode[stbir__encode_order2] + 0.5f; + float f3 = encode[stbir__encode_order3] + 0.5f; STBIR_SIMD_NO_UNROLL(encode); - f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f; - f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f; - f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f; - f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f; + STBIR_CLAMP(f0, 0, 65535); output[0-4] = (unsigned short)f0; + STBIR_CLAMP(f1, 0, 65535); output[1-4] = (unsigned short)f1; + STBIR_CLAMP(f2, 0, 65535); output[2-4] = (unsigned short)f2; + STBIR_CLAMP(f3, 0, 65535); output[3-4] = (unsigned short)f3; output += 4; encode += 4; } @@ -9444,14 +11597,20 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - float f; - STBIR_NO_UNROLL(encode); - f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f; + float f0 = encode[stbir__encode_order0] + 0.5f; #if stbir__coder_min_num >= 2 - f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f; + float f1 = encode[stbir__encode_order1] + 0.5f; #endif #if stbir__coder_min_num >= 3 - f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f; + float f2 = encode[stbir__encode_order2] + 0.5f; + #endif + STBIR_NO_UNROLL(encode); + STBIR_CLAMP(f0, 0, 65535); output[0] = (unsigned short)f0; + #if stbir__coder_min_num >= 2 + STBIR_CLAMP(f1, 0, 65535); output[1] = (unsigned short)f1; + #endif + #if stbir__coder_min_num >= 3 + STBIR_CLAMP(f2, 0, 65535); output[2] = (unsigned short)f2; #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -9459,7 +11618,7 @@ static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int #endif } -static float * STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { float STBIR_STREAMOUT_PTR( * ) decode = decodep; float * decode_end = (float*) decode + width_times_channels; @@ -9546,51 +11705,41 @@ static float * STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decod return decode_end; } -static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { stbir__FP16 STBIR_SIMD_STREAMOUT_PTR( * ) output = (stbir__FP16*) outputp; stbir__FP16 * end_output = ( (stbir__FP16*) output ) + width_times_channels; #ifdef STBIR_SIMD - if ( width_times_channels >= 8 ) + // double width blocks while they fit, then the four-at-a-time and scalar loops below finish the row. (the encode can + // be in place - with an output callback it converts into the float buffer - so no backing up to redo the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= 8 ) { - float const * end_encode_m8 = encode + width_times_channels - 8; - end_output -= 8; - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) + STBIR_SIMD_NO_UNROLL(encode); + #ifdef stbir__decode_swizzle + #ifdef STBIR_SIMD8 { - STBIR_SIMD_NO_UNROLL(encode); - #ifdef stbir__decode_swizzle - #ifdef STBIR_SIMD8 - { - stbir__simdf8 of; - stbir__simdf8_load( of, encode ); - stbir__encode_simdf8_unflip( of ); - stbir__float_to_half_SIMD( output, (float*)&of ); - } - #else - { - stbir__simdf of[2]; - stbir__simdf_load( of[0], encode ); - stbir__simdf_load( of[1], encode+4 ); - stbir__encode_simdf4_unflip( of[0] ); - stbir__encode_simdf4_unflip( of[1] ); - stbir__float_to_half_SIMD( output, (float*)of ); - } - #endif - #else - stbir__float_to_half_SIMD( output, encode ); - #endif - encode += 8; - output += 8; - if ( output <= end_output ) - continue; - if ( output == ( end_output + 8 ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; + stbir__simdf8 of; + stbir__simdf8_load( of, encode ); + stbir__encode_simdf8_unflip( of ); + stbir__float_to_half_SIMD( output, (float*)&of ); } - return; + #else + { + stbir__simdf of[2]; + stbir__simdf_load( of[0], encode ); + stbir__simdf_load( of[1], encode+4 ); + stbir__encode_simdf4_unflip( of[0] ); + stbir__encode_simdf4_unflip( of[1] ); + stbir__float_to_half_SIMD( output, (float*)of ); + } + #endif + #else + stbir__float_to_half_SIMD( output, encode ); + #endif + encode += 8; + output += 8; } #endif @@ -9600,11 +11749,16 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { + float e0, e1, e2, e3; STBIR_SIMD_NO_UNROLL(output); - output[0-4] = stbir__float_to_half(encode[stbir__encode_order0]); - output[1-4] = stbir__float_to_half(encode[stbir__encode_order1]); - output[2-4] = stbir__float_to_half(encode[stbir__encode_order2]); - output[3-4] = stbir__float_to_half(encode[stbir__encode_order3]); + e0 = encode[stbir__encode_order0]; + e1 = encode[stbir__encode_order1]; + e2 = encode[stbir__encode_order2]; + e3 = encode[stbir__encode_order3]; + output[0-4] = stbir__float_to_half(e0); + output[1-4] = stbir__float_to_half(e1); + output[2-4] = stbir__float_to_half(e2); + output[3-4] = stbir__float_to_half(e3); output += 4; encode += 4; } @@ -9616,13 +11770,27 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - STBIR_NO_UNROLL(output); - output[0] = stbir__float_to_half(encode[stbir__encode_order0]); + float e0; #if stbir__coder_min_num >= 2 - output[1] = stbir__float_to_half(encode[stbir__encode_order1]); + float e1; #endif #if stbir__coder_min_num >= 3 - output[2] = stbir__float_to_half(encode[stbir__encode_order2]); + float e2; + #endif + STBIR_NO_UNROLL(output); + e0 = encode[stbir__encode_order0]; + #if stbir__coder_min_num >= 2 + e1 = encode[stbir__encode_order1]; + #endif + #if stbir__coder_min_num >= 3 + e2 = encode[stbir__encode_order2]; + #endif + output[0] = stbir__float_to_half(e0); + #if stbir__coder_min_num >= 2 + output[1] = stbir__float_to_half(e1); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__float_to_half(e2); #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -9630,7 +11798,7 @@ static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp #endif } -static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int width_times_channels, void const * inputp ) +static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, ptrdiff_t width_times_channels, void const * inputp ) { #ifdef stbir__decode_swizzle float STBIR_STREAMOUT_PTR( * ) decode = decodep; @@ -9646,7 +11814,6 @@ static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, i for(;;) { STBIR_NO_UNROLL(decode); - #ifdef stbir__decode_swizzle #ifdef STBIR_SIMD8 { stbir__simdf8 of0,of1; @@ -9674,7 +11841,6 @@ static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, i stbir__simdf_store( decode+12, of3 ); } #endif - #endif decode += 16; input += 16; if ( decode <= decode_end ) @@ -9734,7 +11900,7 @@ static float * STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, i #endif } -static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int width_times_channels, float const * encode ) +static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, ptrdiff_t width_times_channels, float const * encode ) { #if !defined( STBIR_FLOAT_HIGH_CLAMP ) && !defined(STBIR_FLOAT_LOW_CLAMP) && !defined(stbir__decode_swizzle) @@ -9766,39 +11932,29 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int const stbir__simdfX low_clamp = stbir__simdf_frepX(STBIR_FLOAT_LOW_CLAMP); #endif - if ( width_times_channels >= ( stbir__simdfX_float_count * 2 ) ) + // do as many double width blocks as fit and let the four-at-a-time and scalar loops below finish the row. (with an + // output callback we encode in place, so we can't use the usual trick of backing up and redoing the last block) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( end_output - output ) >= ( stbir__simdfX_float_count * 2 ) ) { - float const * end_encode_m8 = encode + width_times_channels - ( stbir__simdfX_float_count * 2 ); - end_output -= ( stbir__simdfX_float_count * 2 ); - STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR - for(;;) - { - stbir__simdfX e0, e1; - STBIR_SIMD_NO_UNROLL(encode); - stbir__simdfX_load( e0, encode ); - stbir__simdfX_load( e1, encode+stbir__simdfX_float_count ); + stbir__simdfX e0, e1; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_load( e0, encode ); + stbir__simdfX_load( e1, encode+stbir__simdfX_float_count ); #ifdef STBIR_FLOAT_HIGH_CLAMP - stbir__simdfX_min( e0, e0, high_clamp ); - stbir__simdfX_min( e1, e1, high_clamp ); + stbir__simdfX_min( e0, e0, high_clamp ); + stbir__simdfX_min( e1, e1, high_clamp ); #endif #ifdef STBIR_FLOAT_LOW_CLAMP - stbir__simdfX_max( e0, e0, low_clamp ); - stbir__simdfX_max( e1, e1, low_clamp ); + stbir__simdfX_max( e0, e0, low_clamp ); + stbir__simdfX_max( e1, e1, low_clamp ); #endif - stbir__encode_simdfX_unflip( e0 ); - stbir__encode_simdfX_unflip( e1 ); - stbir__simdfX_store( output, e0 ); - stbir__simdfX_store( output+stbir__simdfX_float_count, e1 ); - encode += stbir__simdfX_float_count * 2; - output += stbir__simdfX_float_count * 2; - if ( output <= end_output ) - continue; - if ( output == ( end_output + ( stbir__simdfX_float_count * 2 ) ) ) - break; - output = end_output; // backup and do last couple - encode = end_encode_m8; - } - return; + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_store( output, e0 ); + stbir__simdfX_store( output+stbir__simdfX_float_count, e1 ); + encode += stbir__simdfX_float_count * 2; + output += stbir__simdfX_float_count * 2; } // try to do blocks of 4 when you can @@ -9811,10 +11967,10 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int STBIR_NO_UNROLL(encode); stbir__simdf_load( e0, encode ); #ifdef STBIR_FLOAT_HIGH_CLAMP - stbir__simdf_min( e0, e0, high_clamp ); + stbir__simdf_min( e0, e0, stbir__simdfX_cast_to_simdf4( high_clamp ) ); #endif #ifdef STBIR_FLOAT_LOW_CLAMP - stbir__simdf_max( e0, e0, low_clamp ); + stbir__simdf_max( e0, e0, stbir__simdfX_cast_to_simdf4( low_clamp ) ); #endif stbir__encode_simdf4_unflip( e0 ); stbir__simdf_store( output-4, e0 ); @@ -9832,12 +11988,24 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int STBIR_SIMD_NO_UNROLL_LOOP_START while( output <= end_output ) { - float e; + // read everything before writing anything - with an output callback we encode in place, so this is an in-place unflip + float e0 = encode[ stbir__encode_order0 ]; + float e1 = encode[ stbir__encode_order1 ]; + float e2 = encode[ stbir__encode_order2 ]; + float e3 = encode[ stbir__encode_order3 ]; STBIR_SIMD_NO_UNROLL(encode); - e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0-4] = e; - e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1-4] = e; - e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2-4] = e; - e = encode[ stbir__encode_order3 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[3-4] = e; + stbir_scalar_hi_clamp( e0 ); + stbir_scalar_lo_clamp( e0 ); + stbir_scalar_hi_clamp( e1 ); + stbir_scalar_lo_clamp( e1 ); + stbir_scalar_hi_clamp( e2 ); + stbir_scalar_lo_clamp( e2 ); + stbir_scalar_hi_clamp( e3 ); + stbir_scalar_lo_clamp( e3 ); + output[0-4] = e0; + output[1-4] = e1; + output[2-4] = e2; + output[3-4] = e3; output += 4; encode += 4; } @@ -9852,14 +12020,20 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int STBIR_NO_UNROLL_LOOP_START while( output < end_output ) { - float e; - STBIR_NO_UNROLL(encode); - e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0] = e; + float e0 = encode[ stbir__encode_order0 ]; #if stbir__coder_min_num >= 2 - e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1] = e; + float e1 = encode[ stbir__encode_order1 ]; #endif #if stbir__coder_min_num >= 3 - e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2] = e; + float e2 = encode[ stbir__encode_order2 ]; + #endif + STBIR_NO_UNROLL(encode); + stbir_scalar_hi_clamp( e0 ); stbir_scalar_lo_clamp( e0 ); output[0] = e0; + #if stbir__coder_min_num >= 2 + stbir_scalar_hi_clamp( e1 ); stbir_scalar_lo_clamp( e1 ); output[1] = e1; + #endif + #if stbir__coder_min_num >= 3 + stbir_scalar_hi_clamp( e2 ); stbir_scalar_lo_clamp( e2 ); output[2] = e2; #endif output += stbir__coder_min_num; encode += stbir__coder_min_num; @@ -9871,6 +12045,14 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int #undef stbir__decode_suffix #undef stbir__decode_simdf8_flip +#undef stbir__decode_simdf8_flip_i +#undef stbir__decode_simdf4_flip_i +#undef stbir__decode_simdi_flip_u8 +#undef stbir__decode_simdi_flip_u16 +#ifdef STBIR__SWZ8 +#undef STBIR__SWZ8 +#undef STBIR__SWZ16 +#endif #undef stbir__decode_simdf4_flip #undef stbir__decode_order0 #undef stbir__decode_order1 @@ -9881,6 +12063,9 @@ static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int #undef stbir__encode_order2 #undef stbir__encode_order3 #undef stbir__encode_simdf8_unflip +#undef stbir__encode_simdf16_unflip +#undef stbir__decode_simdf16_flip_i +#undef stbir__decode_simdi8_flip_u16 #undef stbir__encode_simdf4_unflip #undef stbir__encode_simdfX_unflip #undef STBIR__CODER_NAME @@ -10013,7 +12198,14 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output #endif input += (4*stbir__simdfX_float_count); - stbIF0( output0 += (4*stbir__simdfX_float_count); ) stbIF1( output1 += (4*stbir__simdfX_float_count); ) stbIF2( output2 += (4*stbir__simdfX_float_count); ) stbIF3( output3 += (4*stbir__simdfX_float_count); ) stbIF4( output4 += (4*stbir__simdfX_float_count); ) stbIF5( output5 += (4*stbir__simdfX_float_count); ) stbIF6( output6 += (4*stbir__simdfX_float_count); ) stbIF7( output7 += (4*stbir__simdfX_float_count); ) + stbIF0( output0 += (4*stbir__simdfX_float_count); ) + stbIF1( output1 += (4*stbir__simdfX_float_count); ) + stbIF2( output2 += (4*stbir__simdfX_float_count); ) + stbIF3( output3 += (4*stbir__simdfX_float_count); ) + stbIF4( output4 += (4*stbir__simdfX_float_count); ) + stbIF5( output5 += (4*stbir__simdfX_float_count); ) + stbIF6( output6 += (4*stbir__simdfX_float_count); ) + stbIF7( output7 += (4*stbir__simdfX_float_count); ) } STBIR_SIMD_NO_UNROLL_LOOP_START while ( ( (char*)input_end - (char*) input ) >= 16 ) @@ -10024,27 +12216,34 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output stbir__simdf_load( r0, input ); #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE - stbIF0( stbir__simdf_load( o0, output0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) - stbIF1( stbir__simdf_load( o0, output1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) - stbIF2( stbir__simdf_load( o0, output2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) - stbIF3( stbir__simdf_load( o0, output3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) - stbIF4( stbir__simdf_load( o0, output4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) - stbIF5( stbir__simdf_load( o0, output5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) - stbIF6( stbir__simdf_load( o0, output6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) - stbIF7( stbir__simdf_load( o0, output7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) + stbIF0( stbir__simdf_load( o0, output0 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) + stbIF1( stbir__simdf_load( o0, output1 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) + stbIF2( stbir__simdf_load( o0, output2 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) + stbIF3( stbir__simdf_load( o0, output3 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) + stbIF4( stbir__simdf_load( o0, output4 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) + stbIF5( stbir__simdf_load( o0, output5 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) + stbIF6( stbir__simdf_load( o0, output6 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) + stbIF7( stbir__simdf_load( o0, output7 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) #else - stbIF0( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) - stbIF1( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) - stbIF2( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) - stbIF3( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) - stbIF4( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) - stbIF5( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) - stbIF6( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) - stbIF7( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) + stbIF0( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) + stbIF1( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) + stbIF2( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) + stbIF3( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) + stbIF4( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) + stbIF5( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) + stbIF6( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) + stbIF7( stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) #endif input += 4; - stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; ) + stbIF0( output0 += 4; ) + stbIF1( output1 += 4; ) + stbIF2( output2 += 4; ) + stbIF3( output3 += 4; ) + stbIF4( output4 += 4; ) + stbIF5( output5 += 4; ) + stbIF6( output6 += 4; ) + stbIF7( output7 += 4; ) } } #else @@ -10077,7 +12276,14 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output #endif input += 4; - stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; ) + stbIF0( output0 += 4; ) + stbIF1( output1 += 4; ) + stbIF2( output2 += 4; ) + stbIF3( output3 += 4; ) + stbIF4( output4 += 4; ) + stbIF5( output5 += 4; ) + stbIF6( output6 += 4; ) + stbIF7( output7 += 4; ) } #endif STBIR_NO_UNROLL_LOOP_START @@ -10107,7 +12313,14 @@ static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** output #endif ++input; - stbIF0( ++output0; ) stbIF1( ++output1; ) stbIF2( ++output2; ) stbIF3( ++output3; ) stbIF4( ++output4; ) stbIF5( ++output5; ) stbIF6( ++output6; ) stbIF7( ++output7; ) + stbIF0( ++output0; ) + stbIF1( ++output1; ) + stbIF2( ++output2; ) + stbIF3( ++output3; ) + stbIF4( ++output4; ) + stbIF5( ++output5; ) + stbIF6( ++output6; ) + stbIF7( ++output7; ) } } @@ -10125,7 +12338,7 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF7( float const * input7 = inputs[7]; float c7s = vertical_coefficients[7]; ) #if ( STBIR__vertical_channels == 1 ) && !defined(STB_IMAGE_RESIZE_VERTICAL_CONTINUE) - // check single channel one weight + // one coefficient within a millionth of 1 (any channel count, the single coefficient instantiation): a copy if ( ( c0s >= (1.0f-0.000001f) ) && ( c0s <= (1.0f+0.000001f) ) ) { STBIR_MEMCPY( output, input0, (char*)input0_end - (char*)input0 ); @@ -10186,7 +12399,14 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbir__simdfX_store( output, o0 ); stbir__simdfX_store( output+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output+(3*stbir__simdfX_float_count), o3 ); output += (4*stbir__simdfX_float_count); - stbIF0( input0 += (4*stbir__simdfX_float_count); ) stbIF1( input1 += (4*stbir__simdfX_float_count); ) stbIF2( input2 += (4*stbir__simdfX_float_count); ) stbIF3( input3 += (4*stbir__simdfX_float_count); ) stbIF4( input4 += (4*stbir__simdfX_float_count); ) stbIF5( input5 += (4*stbir__simdfX_float_count); ) stbIF6( input6 += (4*stbir__simdfX_float_count); ) stbIF7( input7 += (4*stbir__simdfX_float_count); ) + stbIF0( input0 += (4*stbir__simdfX_float_count); ) + stbIF1( input1 += (4*stbir__simdfX_float_count); ) + stbIF2( input2 += (4*stbir__simdfX_float_count); ) + stbIF3( input3 += (4*stbir__simdfX_float_count); ) + stbIF4( input4 += (4*stbir__simdfX_float_count); ) + stbIF5( input5 += (4*stbir__simdfX_float_count); ) + stbIF6( input6 += (4*stbir__simdfX_float_count); ) + stbIF7( input7 += (4*stbir__simdfX_float_count); ) } STBIR_SIMD_NO_UNROLL_LOOP_START @@ -10196,21 +12416,28 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, STBIR_SIMD_NO_UNROLL(output); #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE - stbIF0( stbir__simdf_load( o0, output ); stbir__simdf_load( r0, input0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); ) + stbIF0( stbir__simdf_load( o0, output ); stbir__simdf_load( r0, input0 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c0 ) ); ) #else - stbIF0( stbir__simdf_load( r0, input0 ); stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); ) + stbIF0( stbir__simdf_load( r0, input0 ); stbir__simdf_mult( o0, r0, stbir__simdfX_cast_to_simdf4( c0 ) ); ) #endif - stbIF1( stbir__simdf_load( r0, input1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); ) - stbIF2( stbir__simdf_load( r0, input2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); ) - stbIF3( stbir__simdf_load( r0, input3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); ) - stbIF4( stbir__simdf_load( r0, input4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); ) - stbIF5( stbir__simdf_load( r0, input5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); ) - stbIF6( stbir__simdf_load( r0, input6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); ) - stbIF7( stbir__simdf_load( r0, input7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); ) + stbIF1( stbir__simdf_load( r0, input1 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c1 ) ); ) + stbIF2( stbir__simdf_load( r0, input2 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c2 ) ); ) + stbIF3( stbir__simdf_load( r0, input3 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c3 ) ); ) + stbIF4( stbir__simdf_load( r0, input4 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c4 ) ); ) + stbIF5( stbir__simdf_load( r0, input5 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c5 ) ); ) + stbIF6( stbir__simdf_load( r0, input6 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c6 ) ); ) + stbIF7( stbir__simdf_load( r0, input7 ); stbir__simdf_madd( o0, o0, r0, stbir__simdfX_cast_to_simdf4( c7 ) ); ) stbir__simdf_store( output, o0 ); output += 4; - stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; ) + stbIF0( input0 += 4; ) + stbIF1( input1 += 4; ) + stbIF2( input2 += 4; ) + stbIF3( input3 += 4; ) + stbIF4( input4 += 4; ) + stbIF5( input5 += 4; ) + stbIF6( input6 += 4; ) + stbIF7( input7 += 4; ) } } #else @@ -10233,7 +12460,14 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF7( o0 += input7[0] * c7s; o1 += input7[1] * c7s; o2 += input7[2] * c7s; o3 += input7[3] * c7s; ) output[0] = o0; output[1] = o1; output[2] = o2; output[3] = o3; output += 4; - stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; ) + stbIF0( input0 += 4; ) + stbIF1( input1 += 4; ) + stbIF2( input2 += 4; ) + stbIF3( input3 += 4; ) + stbIF4( input4 += 4; ) + stbIF5( input5 += 4; ) + stbIF6( input6 += 4; ) + stbIF7( input7 += 4; ) } #endif STBIR_NO_UNROLL_LOOP_START @@ -10255,7 +12489,14 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, stbIF7( o0 += input7[0] * c7s; ) output[0] = o0; ++output; - stbIF0( ++input0; ) stbIF1( ++input1; ) stbIF2( ++input2; ) stbIF3( ++input3; ) stbIF4( ++input4; ) stbIF5( ++input5; ) stbIF6( ++input6; ) stbIF7( ++input7; ) + stbIF0( ++input0; ) + stbIF1( ++input1; ) + stbIF2( ++input2; ) + stbIF3( ++input3; ) + stbIF4( ++input4; ) + stbIF5( ++input5; ) + stbIF6( ++input6; ) + stbIF7( ++input7; ) } } @@ -10325,95 +12566,163 @@ static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, #define stbir__store_output_tiny stbir__store_output #endif -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_1_coeff)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_1_coeff)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__1_coeff_only(); stbir__store_output_tiny(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_2_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_2_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_2taps(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__2_coeff_only(); stbir__store_output_tiny(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_3_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_3_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_finish3(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__3_coeff_only(); stbir__store_output_tiny(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__store_output(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_rem1(4); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__1_coeff_remnant(4); stbir__store_output(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_6_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_6_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_rem2(4); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__2_coeff_remnant(4); stbir__store_output(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_rem3(4); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__3_coeff_remnant(4); @@ -10421,28 +12730,53 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( flo } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__4_coeff_continue_from_4(4); stbir__store_output(); } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_rem1(8); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__4_coeff_continue_from_4(4); stbir__1_coeff_remnant(8); @@ -10450,14 +12784,27 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( flo } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_rem2(8); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__4_coeff_continue_from_4(4); stbir__2_coeff_remnant(8); @@ -10465,15 +12812,28 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( fl } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_rem3(8); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__4_coeff_continue_from_4(4); stbir__3_coeff_remnant(8); @@ -10481,14 +12841,27 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( fl } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + float const * hc = column->coefficients; stbir__4_coeff_start(); stbir__4_coeff_continue_from_4(4); stbir__4_coeff_continue_from_4(8); @@ -10496,15 +12869,168 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( fl } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_13_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_rem1(12); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + float const * hc = column->coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__1_coeff_remnant(12); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_14_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_rem2(12); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + float const * hc = column->coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__2_coeff_remnant(12); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_15_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + stbir__3_coeff_setup(); + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_rem3(12); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + float const * hc = column->coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__3_coeff_remnant(12); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_16_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_cont(12); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + float const * hc = column->coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__4_coeff_continue_from_4(12); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_17_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + #ifdef stbir__x4_start + // four outputs at a time while they last (one channel only, see the stbir__x4 macros) + while ( ( output_end - output ) >= 4 ) + { + stbir__x4_setup(); + stbir__x4_start(); + stbir__x4_cont(4); + stbir__x4_cont(8); + stbir__x4_cont(12); + stbir__x4_rem1(16); + stbir__x4_finish(); + stbir__x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + float const * hc = column->coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__4_coeff_continue_from_4(12); + stbir__1_coeff_remnant(16); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 4 + 3 ) >> 2; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + int n = ( column->width - 4 + 3 ) >> 2; + float const * hc = column->coefficients; stbir__4_coeff_start(); STBIR_SIMD_NO_UNROLL_LOOP_START @@ -10518,15 +13044,15 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 5 + 3 ) >> 2; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + int n = ( column->width - 5 + 3 ) >> 2; + float const * hc = column->coefficients; stbir__4_coeff_start(); STBIR_SIMD_NO_UNROLL_LOOP_START @@ -10541,15 +13067,15 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 6 + 3 ) >> 2; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + int n = ( column->width - 6 + 3 ) >> 2; + float const * hc = column->coefficients; stbir__4_coeff_start(); STBIR_SIMD_NO_UNROLL_LOOP_START @@ -10565,16 +13091,16 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 } while ( output < output_end ); } -static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) { - float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; stbir__3_coeff_setup(); STBIR_SIMD_NO_UNROLL_LOOP_START do { - float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; - int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 7 + 3 ) >> 2; - float const * hc = horizontal_coefficients; + float const * decode = decode_buffer; + int n = ( column->width - 7 + 3 ) >> 2; + float const * hc = column->coefficients; stbir__4_coeff_start(); STBIR_SIMD_NO_UNROLL_LOOP_START @@ -10590,6 +13116,304 @@ static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 } while ( output < output_end ); } +// polyphase period 1 kernels (integer downsamples, see stbir__resample_horizontal_gather): every output uses the same +// coefficient set, so it is loaded into registers once per run and the pixel loop is just the decode loads and +// multiply-adds. the same steps as the kernels above, for the widths integer downsamples produce (4, 8: 2:1 with the +// support 1 and 2 filters; 5, 9: the same one tap wider; 11/12/13 and 16/17: 3:1 and 4:1 with the support 2 filters - +// 3:1 trims to 11 because its outermost taps land exactly on the filter's zero at +-2). +// only for the channel counts that define the step macros. +#ifdef stbir__period1_setup_4 +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_4() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_5() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_remnant(4); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_1_coeff_remnant(4); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_8() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_9() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_remnant(8); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_1_coeff_remnant(8); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_11() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_remnant3(8); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_3_coeff_remnant(8); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_12() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_continue2(); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_4_coeff_continue2(); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_13_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_13() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_continue2(); + stbir__period1_x4_remnant(12); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_4_coeff_continue2(); + stbir__period1_1_coeff_remnant(12); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_16_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_16() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_continue2(); + stbir__period1_x4_continue3(); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_4_coeff_continue2(); + stbir__period1_4_coeff_continue3(); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_17_coeffs_period1)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__column_entry const * column, int entry_stride ) +{ + float const * output_end = output_buffer + (ptrdiff_t)output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + float const * hc = column->coefficients; + int advance = column->advance; + stbir__period1_setup_17() + STBIR__UNUSED( entry_stride ); + #ifdef stbir__period1_x4_start + // four outputs at a time while they last (one channel only) + while ( ( output_end - output ) >= 4 ) + { + stbir__period1_x4_setup(); + stbir__period1_x4_start(); + stbir__period1_x4_continue(); + stbir__period1_x4_continue2(); + stbir__period1_x4_continue3(); + stbir__period1_x4_remnant(16); + stbir__period1_x4_store(); + } + if ( output < output_end ) + #endif + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer; + stbir__period1_4_coeff_start(); + stbir__period1_4_coeff_continue(); + stbir__period1_4_coeff_continue2(); + stbir__period1_4_coeff_continue3(); + stbir__period1_1_coeff_remnant(16); + stbir__period1_store_output(); + } while ( output < output_end ); +} + +// by coefficient count (null = use the normal kernel over the one entry) +static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_period1_funcs)[18]= +{ + 0, 0, 0, 0, + STBIR_chans(stbir__horizontal_gather_,_channels_with_4_coeffs_period1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_5_coeffs_period1), + 0, 0, + STBIR_chans(stbir__horizontal_gather_,_channels_with_8_coeffs_period1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_9_coeffs_period1), + 0, + STBIR_chans(stbir__horizontal_gather_,_channels_with_11_coeffs_period1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_12_coeffs_period1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_13_coeffs_period1), + 0, 0, + STBIR_chans(stbir__horizontal_gather_,_channels_with_16_coeffs_period1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_17_coeffs_period1) +}; +#else +static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_period1_funcs)[18]= { 0 }; +#endif + static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_funcs)[4]= { STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod0), @@ -10598,7 +13422,7 @@ static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_ga STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod3), }; -static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_funcs)[12]= +static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_funcs)[17]= { STBIR_chans(stbir__horizontal_gather_,_channels_with_1_coeff), STBIR_chans(stbir__horizontal_gather_,_channels_with_2_coeffs), @@ -10612,10 +13436,56 @@ static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_ga STBIR_chans(stbir__horizontal_gather_,_channels_with_10_coeffs), STBIR_chans(stbir__horizontal_gather_,_channels_with_11_coeffs), STBIR_chans(stbir__horizontal_gather_,_channels_with_12_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_13_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_14_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_15_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_16_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_17_coeffs), }; +#ifdef stbir__period1_setup_4 +#undef stbir__period1_setup_4 +#undef stbir__period1_setup_5 +#undef stbir__period1_setup_8 +#undef stbir__period1_setup_9 +#undef stbir__period1_setup_11 +#undef stbir__period1_setup_12 +#undef stbir__period1_setup_13 +#undef stbir__period1_setup_16 +#undef stbir__period1_setup_17 +#undef stbir__period1_4_coeff_start +#undef stbir__period1_4_coeff_continue +#undef stbir__period1_4_coeff_continue2 +#undef stbir__period1_4_coeff_continue3 +#undef stbir__period1_1_coeff_remnant +#undef stbir__period1_3_coeff_remnant +#undef stbir__period1_store_output +#endif +#ifdef stbir__period1_x4_start +#undef stbir__period1_x4_setup +#undef stbir__period1_x4_start +#undef stbir__period1_x4_next +#undef stbir__period1_x4_continue +#undef stbir__period1_x4_continue2 +#undef stbir__period1_x4_continue3 +#undef stbir__period1_x4_remnant +#undef stbir__period1_x4_remnant3 +#undef stbir__period1_x4_store +#endif #undef STBIR__horizontal_channels #undef STB_IMAGE_RESIZE_DO_HORIZONTALS +#ifdef stbir__x4_start +#undef stbir__x4_setup +#undef stbir__x4_start +#undef stbir__x4_cont +#undef stbir__x4_rem1 +#undef stbir__x4_rem2 +#undef stbir__x4_rem3 +#undef stbir__x4_2taps +#undef stbir__x4_finish +#undef stbir__x4_finish3 +#undef stbir__x4_store +#endif #undef stbir__1_coeff_only #undef stbir__1_coeff_remnant #undef stbir__2_coeff_only