Use zstd dict for packing/unpacking frame images.

This only affects run-time memory usage and needs an offline calculation of
the dictionary. Results vary depending on similarity of image blocks.

agora        34.96 MB ->  28.21 MB
agora2       40.75 MB ->  34.14 MB
android-vk   36.21 MB ->  18.44 MB
astar3       44.72 MB ->  43.38 MB
clipper1    134.36 MB ->  52.16 MB
fi           50.82 MB ->  40.79 MB
fi-big      537.74 MB -> 469.54 MB
test         23.26 MB ->   1.87 MB
This commit is contained in:
Bartosz Taudul
2021-05-15 17:02:25 +02:00
parent d555256546
commit c91c7a7fd5
3 changed files with 43 additions and 3 deletions

View File

@@ -1291,6 +1291,18 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks )
if( eventMask & EventType::FrameImages )
{
ZSTD_CDict* cdict = nullptr;
if( fileVer >= FileVersion( 0, 7, 8 ) )
{
uint32_t dsz;
f.Read( dsz );
auto dict = new char[dsz];
f.Read( dict, dsz );
cdict = ZSTD_createCDict( dict, dsz, 3 );
m_texcomp.SetDict( ZSTD_createDDict( dict, dsz ) );
delete[] dict;
}
f.Read( sz );
m_data.frameImage.reserve_exact( sz, m_slab );
s_loadProgress.subTotal.store( sz, std::memory_order_relaxed );
@@ -1353,9 +1365,16 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks )
data[idx].fi = fi;
data[idx].state.store( JobData::InProgress, std::memory_order_release );
td->Queue( [this, &data, idx, fi, fileVer] {
td->Queue( [this, &data, idx, fi, fileVer, cdict] {
if( fileVer <= FileVersion( 0, 6, 9 ) ) m_texcomp.Rdo( data[idx].buf, fi->w * fi->h / 16 );
fi->csz = m_texcomp.Pack( data[idx].ctx, data[idx].outbuf, data[idx].outsz, data[idx].buf, fi->w * fi->h / 2 );
if( cdict )
{
fi->csz = m_texcomp.Pack( data[idx].ctx, cdict, data[idx].outbuf, data[idx].outsz, data[idx].buf, fi->w * fi->h / 2 );
}
else
{
fi->csz = m_texcomp.Pack( data[idx].ctx, data[idx].outbuf, data[idx].outsz, data[idx].buf, fi->w * fi->h / 2 );
}
data[idx].state.store( JobData::DataReady, std::memory_order_release );
} );
@@ -1387,9 +1406,17 @@ Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks )
}
}
}
ZSTD_freeCDict( cdict );
}
else
{
if( fileVer >= FileVersion( 0, 7, 8 ) )
{
uint32_t dsz;
f.Read( dsz );
f.Skip( dsz );
}
f.Read( sz );
s_loadProgress.subTotal.store( sz, std::memory_order_relaxed );
for( uint64_t i=0; i<sz; i++ )