Scarfski/basis_universal
1
0
Fork 0
mirror of https://github.com/BinomialLLC/basis_universal.git synced 2026-11-06 08:05:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Huffman encoder fixes for very large streams

Browse Source 
Allowing Huffman decoder to use different fast bits table sizes
Adding NV and AMD BC1 decoders
This commit is contained in:
richgel999
2020-05-05 23:10:21 -04:00
parent bb64ec604c
commit a1689504ea
6 changed files with 192 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
basisu_enc.cpp
+15 -6
View File
@@ -678,7 +678,7 @@ namespace basisu
if ((s >= num_syms) || (A[r].m_key < A[s].m_key))
{
A[next].m_key = A[r].m_key;
A[r].m_key = static_cast<uint16_t>(next);
A[r].m_key = next;
++r;
}
else
@@ -689,13 +689,13 @@ namespace basisu
if ((s >= num_syms) || ((r < next) && A[r].m_key < A[s].m_key))
{
A[next].m_key = static_cast<uint16_t>(A[next].m_key + A[r].m_key);
A[r].m_key = static_cast<uint16_t>(next);
A[next].m_key = A[next].m_key + A[r].m_key;
A[r].m_key = next;
++r;
}
else
{
A[next].m_key = static_cast<uint16_t>(A[next].m_key + A[s].m_key);
A[next].m_key = A[next].m_key + A[s].m_key;
++s;
}
}
@@ -715,7 +715,7 @@ namespace basisu
;
for ( ; num_avail > num_used; --next, --num_avail)
A[next].m_key = static_cast<uint16_t>(depth);
A[next].m_key = depth;
num_avail = 2 * num_used;
num_used = 0;
@@ -763,6 +763,10 @@ namespace basisu
for (i = 0; i < num_syms; i++)
{
uint32_t freq = pSyms0[i].m_key;
// We scale all input frequencies to 16-bits.
assert(freq <= UINT16_MAX);
hist[freq & 0xFF]++;
hist[256 + ((freq >> 8) & 0xFF)]++;
}
@@ -884,8 +888,13 @@ namespace basisu
else
{
for (uint32_t i = 0; i < num_syms; i++)
{
if (pSym_freq[i])
sym_freq[i] = static_cast<uint16_t>(maximum<uint32_t>((pSym_freq[i] * 65534U + (max_freq >> 1)) / max_freq, 1));
{
uint32_t f = static_cast<uint32_t>((static_cast<uint64_t>(pSym_freq[i]) * 65534U + (max_freq >> 1)) / max_freq);
sym_freq[i] = static_cast<uint16_t>(clamp<uint32_t>(f, 1, 65534));
}
}
}
return init(num_syms, &sym_freq[0], max_code_size);
basisu_enc.h
+3 -2
View File
@@ -1927,7 +1927,8 @@ namespace basisu
struct sym_freq
{
uint16_t m_key, m_sym_index;
uint32_t m_key;
uint16_t m_sym_index;
};
sym_freq *canonical_huffman_radix_sort_syms(uint32_t num_syms, sym_freq *pSyms0, sym_freq *pSyms1);
@@ -2008,7 +2009,7 @@ namespace basisu
{
if (m_bit_buffer_size)
{
m_total_bits += 8;
m_total_bits += 8 - (m_bit_buffer_size & 7);
append_byte(static_cast<uint8_t>(m_bit_buffer));
m_bit_buffer = 0;
basisu_gpu_texture.cpp
+151 -4
View File
@@ -95,19 +95,18 @@ namespace basisu
bc1_block::unpack_color(l, r0, g0, b0);
bc1_block::unpack_color(h, r1, g1, b1);
c[0].set_noclamp_rgba(r0, g0, b0, 255);
c[1].set_noclamp_rgba(r1, g1, b1, 255);
bool used_punchthrough = false;
if (l > h)
{
c[0].set_noclamp_rgba(r0, g0, b0, 255);
c[1].set_noclamp_rgba(r1, g1, b1, 255);
c[2].set_noclamp_rgba((r0 * 2 + r1) / 3, (g0 * 2 + g1) / 3, (b0 * 2 + b1) / 3, 255);
c[3].set_noclamp_rgba((r1 * 2 + r0) / 3, (g1 * 2 + g0) / 3, (b1 * 2 + b0) / 3, 255);
}
else
{
c[0].set_noclamp_rgba(r0, g0, b0, 255);
c[1].set_noclamp_rgba(r1, g1, b1, 255);
c[2].set_noclamp_rgba((r0 + r1) / 2, (g0 + g1) / 2, (b0 + b1) / 2, 255);
c[3].set_noclamp_rgba(0, 0, 0, 0);
used_punchthrough = true;
@@ -137,6 +136,142 @@ namespace basisu
return used_punchthrough;
}
bool unpack_bc1_nv(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha)
{
static_assert(sizeof(bc1_block) == 8, "sizeof(bc1_block) == 8");
const bc1_block *pBlock = static_cast<const bc1_block *>(pBlock_bits);
const uint32_t l = pBlock->get_low_color();
const uint32_t h = pBlock->get_high_color();
color_rgba c[4];
int r0 = (l >> 11) & 31;
int g0 = (l >> 5) & 63;
int b0 = l & 31;
int r1 = (h >> 11) & 31;
int g1 = (h >> 5) & 63;
int b1 = h & 31;
c[0].b = (uint8_t)((3 * b0 * 22) / 8);
c[0].g = (uint8_t)((g0 << 2) | (g0 >> 4));
c[0].r = (uint8_t)((3 * r0 * 22) / 8);
c[0].a = 0xFF;
c[1].r = (uint8_t)((3 * r1 * 22) / 8);
c[1].g = (uint8_t)((g1 << 2) | (g1 >> 4));
c[1].b = (uint8_t)((3 * b1 * 22) / 8);
c[1].a = 0xFF;
int gdiff = c[1].g - c[0].g;
bool used_punchthrough = false;
if (l > h)
{
c[2].r = (uint8_t)(((2 * r0 + r1) * 22) / 8);
c[2].g = (uint8_t)(((256 * c[0].g + gdiff/4 + 128 + gdiff * 80) / 256));
c[2].b = (uint8_t)(((2 * b0 + b1) * 22) / 8);
c[2].a = 0xFF;
c[3].r = (uint8_t)(((2 * r1 + r0) * 22) / 8);
c[3].g = (uint8_t)((256 * c[1].g - gdiff/4 + 128 - gdiff * 80) / 256);
c[3].b = (uint8_t)(((2 * b1 + b0) * 22) / 8);
c[3].a = 0xFF;
}
else
{
c[2].r = (uint8_t)(((r0 + r1) * 33) / 8);
c[2].g = (uint8_t)((256 * c[0].g + gdiff/4 + 128 + gdiff * 128) / 256);
c[2].b = (uint8_t)(((b0 + b1) * 33) / 8);
c[2].a = 0xFF;
c[3].set_noclamp_rgba(0, 0, 0, 0);
used_punchthrough = true;
}
if (set_alpha)
{
for (uint32_t y = 0; y < 4; y++, pPixels += 4)
{
pPixels[0] = c[pBlock->get_selector(0, y)];
pPixels[1] = c[pBlock->get_selector(1, y)];
pPixels[2] = c[pBlock->get_selector(2, y)];
pPixels[3] = c[pBlock->get_selector(3, y)];
}
}
else
{
for (uint32_t y = 0; y < 4; y++, pPixels += 4)
{
pPixels[0].set_rgb(c[pBlock->get_selector(0, y)]);
pPixels[1].set_rgb(c[pBlock->get_selector(1, y)]);
pPixels[2].set_rgb(c[pBlock->get_selector(2, y)]);
pPixels[3].set_rgb(c[pBlock->get_selector(3, y)]);
}
}
return used_punchthrough;
}
static inline int interp_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 * 43 + c1 * 21 + 32) >> 6; }
static inline int interp_half_5_6_amd(int c0, int c1) { assert(c0 < 256 && c1 < 256); return (c0 + c1 + 1) >> 1; }
bool unpack_bc1_amd(const void *pBlock_bits, color_rgba *pPixels, bool set_alpha)
{
const bc1_block *pBlock = static_cast<const bc1_block *>(pBlock_bits);
const uint32_t l = pBlock->get_low_color();
const uint32_t h = pBlock->get_high_color();
color_rgba c[4];
uint32_t r0, g0, b0, r1, g1, b1;
bc1_block::unpack_color(l, r0, g0, b0);
bc1_block::unpack_color(h, r1, g1, b1);
c[0].set_noclamp_rgba(r0, g0, b0, 255);
c[1].set_noclamp_rgba(r1, g1, b1, 255);
bool used_punchthrough = false;
if (l > h)
{
c[2].set_noclamp_rgba(interp_5_6_amd(r0, r1), interp_5_6_amd(g0, g1), interp_5_6_amd(b0, b1), 255);
c[3].set_noclamp_rgba(interp_5_6_amd(r1, r0), interp_5_6_amd(g1, g0), interp_5_6_amd(b1, b0), 255);
}
else
{
c[2].set_noclamp_rgba(interp_half_5_6_amd(r0, r1), interp_half_5_6_amd(g0, g1), interp_half_5_6_amd(b0, b1), 255);
c[3].set_noclamp_rgba(0, 0, 0, 0);
used_punchthrough = true;
}
if (set_alpha)
{
for (uint32_t y = 0; y < 4; y++, pPixels += 4)
{
pPixels[0] = c[pBlock->get_selector(0, y)];
pPixels[1] = c[pBlock->get_selector(1, y)];
pPixels[2] = c[pBlock->get_selector(2, y)];
pPixels[3] = c[pBlock->get_selector(3, y)];
}
}
else
{
for (uint32_t y = 0; y < 4; y++, pPixels += 4)
{
pPixels[0].set_rgb(c[pBlock->get_selector(0, y)]);
pPixels[1].set_rgb(c[pBlock->get_selector(1, y)]);
pPixels[2].set_rgb(c[pBlock->get_selector(2, y)]);
pPixels[3].set_rgb(c[pBlock->get_selector(3, y)]);
}
}
return used_punchthrough;
}
struct bc4_block
{
enum { cBC4SelectorBits = 3, cTotalSelectorBytes = 6, cMaxSelectorValues = 8 };
@@ -964,6 +1099,16 @@ namespace basisu
unpack_bc1(pBlock, pPixels, true);
break;
}
case texture_format::cBC1_NV:
{
unpack_bc1_nv(pBlock, pPixels, true);
break;
}
case texture_format::cBC1_AMD:
{
unpack_bc1_amd(pBlock, pPixels, true);
break;
}
case texture_format::cBC3:
{
return unpack_bc3(pBlock, pPixels);
@@ -1234,6 +1379,8 @@ namespace basisu
switch (fmt)
{
case texture_format::cBC1:
case texture_format::cBC1_NV:
case texture_format::cBC1_AMD:
{
internal_fmt = KTX_COMPRESSED_RGB_S3TC_DXT1_EXT;
break;
transcoder/basisu.h
+5 -1
View File
@@ -293,7 +293,7 @@ namespace basisu
enum
{
cHuffmanMaxSupportedCodeSize = 16, cHuffmanMaxSupportedInternalCodeSize = 31,
cHuffmanFastLookupBits = 10, cHuffmanFastLookupSize = 1 << cHuffmanFastLookupBits,
cHuffmanFastLookupBits = 10,
cHuffmanMaxSymsLog2 = 14, cHuffmanMaxSyms = 1 << cHuffmanMaxSymsLog2,
// Small zero runs
@@ -341,6 +341,8 @@ namespace basisu
cETC2_R11_EAC,
cETC2_RG11_EAC,
cUASTC4x4,
cBC1_NV,
cBC1_AMD,
// Uncompressed/raw pixels
cRGBA32,
@@ -359,6 +361,8 @@ namespace basisu
case texture_format::cETC2_RGB:
case texture_format::cETC2_ALPHA:
case texture_format::cBC1:
case texture_format::cBC1_NV:
case texture_format::cBC1_AMD:
case texture_format::cBC4:
case texture_format::cPVRTC1_4_RGB:
case texture_format::cPVRTC1_4_RGBA:
transcoder/basisu_transcoder.cpp
+1 -1
View File
@@ -195,7 +195,7 @@ namespace basist
{
crc = ~crc;
const uint8_t* p = reinterpret_cast<const uint8_t*>(r);
const uint8_t* p = static_cast<const uint8_t*>(r);
for (; size; --size)
{
const uint16_t q = *p++ ^ (crc >> 8);
transcoder/basisu_transcoder_internal.h
+17 -11
View File
@@ -122,7 +122,7 @@ namespace basist
basisu::clear_vector(m_tree);
}
bool init(uint32_t total_syms, const uint8_t *pCode_sizes)
bool init(uint32_t total_syms, const uint8_t *pCode_sizes, uint32_t fast_lookup_bits = basisu::cHuffmanFastLookupBits)
{
if (!total_syms)
{
@@ -133,8 +133,10 @@ namespace basist
m_code_sizes.resize(total_syms);
memcpy(&m_code_sizes[0], pCode_sizes, total_syms);
const uint32_t huffman_fast_lookup_size = 1 << fast_lookup_bits;
m_lookup.resize(0);
m_lookup.resize(basisu::cHuffmanFastLookupSize);
m_lookup.resize(huffman_fast_lookup_size);
m_tree.resize(0);
m_tree.resize(total_syms * 2);
@@ -172,10 +174,10 @@ namespace basist
for (l = code_size; l > 0; l--, cur_code >>= 1)
rev_code = (rev_code << 1) | (cur_code & 1);
if (code_size <= basisu::cHuffmanFastLookupBits)
if (code_size <= fast_lookup_bits)
{
uint32_t k = (code_size << 16) | sym_index;
while (rev_code < basisu::cHuffmanFastLookupSize)
while (rev_code < huffman_fast_lookup_size)
{
if (m_lookup[rev_code] != 0)
{
@@ -190,9 +192,9 @@ namespace basist
}
int tree_cur;
if (0 == (tree_cur = m_lookup[rev_code & (basisu::cHuffmanFastLookupSize - 1)]))
if (0 == (tree_cur = m_lookup[rev_code & (huffman_fast_lookup_size - 1)]))
{
const uint32_t idx = rev_code & (basisu::cHuffmanFastLookupSize - 1);
const uint32_t idx = rev_code & (huffman_fast_lookup_size - 1);
if (m_lookup[idx] != 0)
{
// Supplied codesizes can't create a valid prefix code.
@@ -210,9 +212,9 @@ namespace basist
return false;
}
rev_code >>= (basisu::cHuffmanFastLookupBits - 1);
rev_code >>= (fast_lookup_bits - 1);
for (int j = code_size; j > (basisu::cHuffmanFastLookupBits + 1); j--)
for (int j = code_size; j > ((int)fast_lookup_bits + 1); j--)
{
tree_cur -= ((rev_code >>= 1) & 1);
@@ -260,6 +262,8 @@ namespace basist
}
const basisu::uint8_vec &get_code_sizes() const { return m_code_sizes; }
const basisu::int_vec get_lookup() const { return m_lookup; }
const basisu::int16_vec get_tree() const { return m_tree; }
bool is_valid() const { return m_code_sizes.size() > 0; }
@@ -436,9 +440,11 @@ namespace basist
return v;
}
inline uint32_t decode_huffman(const huffman_decoding_table &ct)
inline uint32_t decode_huffman(const huffman_decoding_table &ct, int fast_lookup_bits = basisu::cHuffmanFastLookupBits)
{
assert(ct.m_code_sizes.size());
const uint32_t huffman_fast_lookup_size = 1 << fast_lookup_bits;
while (m_bit_buf_size < 16)
{
@@ -454,14 +460,14 @@ namespace basist
int code_len;
int sym;
if ((sym = ct.m_lookup[m_bit_buf & (basisu::cHuffmanFastLookupSize - 1)]) >= 0)
if ((sym = ct.m_lookup[m_bit_buf & (huffman_fast_lookup_size - 1)]) >= 0)
{
code_len = sym >> 16;
sym &= 0xFFFF;
}
else
{
code_len = basisu::cHuffmanFastLookupBits;
code_len = fast_lookup_bits;
do
{
sym = ct.m_tree[~sym + ((m_bit_buf >> code_len++) & 1)]; // ~sym = -sym - 1