Merge branch 'branch'

basisu.vcxproj

@@ -205,6 +205,7 @@
     <ClInclude Include="encoder\lodepng.h" />
     <ClInclude Include="transcoder\basisu.h" />
     <ClInclude Include="transcoder\basisu_containers.h" />
+    <ClInclude Include="transcoder\basisu_file_headers.h" />
     <ClInclude Include="transcoder\basisu_transcoder.h" />
     <ClInclude Include="transcoder\basisu_transcoder_internal.h" />
     <ClInclude Include="transcoder\basisu_global_selector_palette.h" />

basisu.vcxproj.filters

@@ -163,6 +163,9 @@
     <ClInclude Include="transcoder\basisu_containers_impl.h">
       <Filter>transcoder</Filter>
     </ClInclude>
+    <ClInclude Include="transcoder\basisu_file_headers.h">
+      <Filter>transcoder</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="transcoder\basisu_transcoder_tables_dxt1_6.inc">

basisu_tool.cpp

@@ -120,6 +120,7 @@ static void print_usage()
 		" -bench: UASTC benchmark mode, for development only\n"
 		" -resample_factor X: Resample all input textures by scale factor X using a box filter\n"
 		" -no_sse: Forbid all SSE instruction set usage\n"
+		" -validate_etc1s: Validate internal ETC1S compressor's data structures.\n"
 		"\n"
 		"Mipmap generation options:\n"
 		" -mipmap: Generate mipmaps for each source image\n"
@@ -406,10 +407,23 @@ public:
 				m_comp_params.m_debug = true;
 				enable_debug_printf(true);
 			}
+			else if (strcasecmp(pArg, "-validate_etc1s") == 0)
+			{
+				m_comp_params.m_validate = true;
+			}
 			else if (strcasecmp(pArg, "-debug_images") == 0)
 				m_comp_params.m_debug_images = true;
 			else if (strcasecmp(pArg, "-stats") == 0)
 				m_comp_params.m_compute_stats = true;
+			else if (strcasecmp(pArg, "-gen_global_codebooks") == 0)
+			{
+			}
+			else if (strcasecmp(pArg, "-use_global_codebooks") == 0)
+			{
+				REMAINING_ARGS_CHECK(1);
+				m_etc1s_use_global_codebooks_file = arg_v[arg_index + 1];
+				arg_count++;
+			}
 			else if (strcasecmp(pArg, "-comp_level") == 0)
 			{
 				REMAINING_ARGS_CHECK(1);
@@ -711,6 +725,7 @@ public:
 
 	std::string m_csv_file;
 
+	std::string m_etc1s_use_global_codebooks_file;
 	bool m_individual;
 	bool m_no_ktx;
 	bool m_etc1_only;
@@ -757,6 +772,55 @@ static bool expand_multifile(command_line_params &opts)
 	return true;
 }
 
+struct basis_data
+{
+	basis_data(basist::etc1_global_selector_codebook& sel_codebook) : 
+		m_transcoder(&sel_codebook) 
+	{
+	}
+	uint8_vec m_file_data;
+	basist::basisu_transcoder m_transcoder;
+};
+static basis_data *load_basis_file(const char *pInput_filename, basist::etc1_global_selector_codebook &sel_codebook, bool force_etc1s)
+{
+	basis_data* p = new basis_data(sel_codebook);
+	uint8_vec &basis_data = p->m_file_data;
+	if (!basisu::read_file_to_vec(pInput_filename, basis_data))
+	{
+		error_printf("Failed reading file \"%s\"\n", pInput_filename);
+		delete p;
+		return nullptr;
+	}
+	printf("Input file \"%s\"\n", pInput_filename);
+	if (!basis_data.size())
+	{
+		error_printf("File is empty!\n");
+		delete p;
+		return nullptr;
+	}
+	if (basis_data.size() > UINT32_MAX)
+	{
+		error_printf("File is too large!\n");
+		delete p;
+		return nullptr;
+	}
+	if (force_etc1s)
+	{
+		if (p->m_transcoder.get_tex_format((const void*)&p->m_file_data[0], (uint32_t)p->m_file_data.size()) != basist::basis_tex_format::cETC1S)
+		{
+			error_printf("Global codebook file must be in ETC1S format!\n");
+			delete p;
+			return nullptr;
+		}
+	}
+	if (!p->m_transcoder.start_transcoding(&basis_data[0], (uint32_t)basis_data.size()))
+	{
+		error_printf("start_transcoding() failed!\n");
+		delete p;
+		return nullptr;
+	}
+	return p;
+}
 static bool compress_mode(command_line_params &opts)
 {
 	basist::etc1_global_selector_codebook sel_codebook(basist::g_global_selector_cb_size, basist::g_global_selector_cb);
@@ -784,13 +848,52 @@ static bool compress_mode(command_line_params &opts)
 		error_printf("No input files to process!\n");
 		return false;
 	}
+	basis_data* pGlobal_codebook_data = nullptr;
+	if (opts.m_etc1s_use_global_codebooks_file.size())
+	{
+		pGlobal_codebook_data = load_basis_file(opts.m_etc1s_use_global_codebooks_file.c_str(), sel_codebook, true);
+		if (!pGlobal_codebook_data)
+			return false;
+#if 0
+		basis_data* pGlobal_codebook_data2 = load_basis_file("xmen_1024.basis", sel_codebook, true);
+		const basist::basisu_lowlevel_etc1s_transcoder &ta = pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder();
+		const basist::basisu_lowlevel_etc1s_transcoder &tb = pGlobal_codebook_data2->m_transcoder.get_lowlevel_etc1s_decoder();
+		if (ta.get_endpoints().size() != tb.get_endpoints().size())
+		{
+			printf("Endpoint CB's don't match\n");
+		}
+		else if (ta.get_selectors().size() != tb.get_selectors().size())
+		{
+			printf("Selector CB's don't match\n");
+		}
+		else
+		{
+			for (uint32_t i = 0; i < ta.get_endpoints().size(); i++)
+			{
+				if (ta.get_endpoints()[i] != tb.get_endpoints()[i])
+				{
+					printf("Endoint CB mismatch entry %u\n", i);
+				}
+			}
+			for (uint32_t i = 0; i < ta.get_selectors().size(); i++)
+			{
+				if (ta.get_selectors()[i] != tb.get_selectors()[i])
+				{
+					printf("Selector CB mismatch entry %u\n", i);
+				}
+			}
+		}
+		delete pGlobal_codebook_data2;
+		pGlobal_codebook_data2 = nullptr;
+#endif
+	}
 						
 	basis_compressor_params &params = opts.m_comp_params;
 
 	params.m_read_source_images = true;
 	params.m_write_output_basis_files = true;
 	params.m_pSel_codebook = &sel_codebook;
-
+	params.m_pGlobal_codebooks = pGlobal_codebook_data ? &pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder() : nullptr; 
 	FILE *pCSV_file = nullptr;
 	if (opts.m_csv_file.size())
 	{
@@ -799,6 +902,7 @@ static bool compress_mode(command_line_params &opts)
 		if (!pCSV_file)
 		{
 			error_printf("Failed opening CVS file \"%s\"\n", opts.m_csv_file.c_str());
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 		fprintf(pCSV_file, "Filename, Size, Slices, Width, Height, HasAlpha, BitsPerTexel, Slice0RGBAvgPSNR, Slice0RGBAAvgPSNR, Slice0Luma709PSNR, Slice0BestETC1SLuma709PSNR, Q, CL, Time, RGBAvgPSNRMin, RGBAvgPSNRAvg, AAvgPSNRMin, AAvgPSNRAvg, Luma709PSNRMin, Luma709PSNRAvg\n");
@@ -865,6 +969,7 @@ static bool compress_mode(command_line_params &opts)
 				pCSV_file = nullptr;
 			}
 
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 
@@ -931,6 +1036,7 @@ static bool compress_mode(command_line_params &opts)
 					pCSV_file = nullptr;
 				}
 
+				delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 				return false;
 			}
 		}
@@ -1009,6 +1115,8 @@ static bool compress_mode(command_line_params &opts)
 		fclose(pCSV_file);
 		pCSV_file = nullptr;
 	}
+	delete pGlobal_codebook_data; 
+	pGlobal_codebook_data = nullptr;
 		
 	return true;
 }
@@ -1018,9 +1126,17 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 	const bool validate_flag = (opts.m_mode == cValidate);
 	basist::etc1_global_selector_codebook sel_codebook(basist::g_global_selector_cb_size, basist::g_global_selector_cb);
 
+	basis_data* pGlobal_codebook_data = nullptr;
+	if (opts.m_etc1s_use_global_codebooks_file.size())
+	{
+		pGlobal_codebook_data = load_basis_file(opts.m_etc1s_use_global_codebooks_file.c_str(), sel_codebook, true);
+		if (!pGlobal_codebook_data)
+			return false;
+	}
 	if (!opts.m_input_filenames.size())
 	{
 		error_printf("No input files to process!\n");
+		delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 		return false;
 	}
 
@@ -1031,6 +1147,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		if (!pCSV_file)
 		{
 			error_printf("Failed opening CVS file \"%s\"\n", opts.m_csv_file.c_str());
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 		//fprintf(pCSV_file, "Filename, Size, Slices, Width, Height, HasAlpha, BitsPerTexel, Slice0RGBAvgPSNR, Slice0RGBAAvgPSNR, Slice0Luma709PSNR, Slice0BestETC1SLuma709PSNR, Q, CL, Time, RGBAvgPSNRMin, RGBAvgPSNRAvg, AAvgPSNRMin, AAvgPSNRAvg, Luma709PSNRMin, Luma709PSNRAvg\n");
@@ -1051,6 +1168,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		{
 			error_printf("Failed reading file \"%s\"\n", pInput_filename);
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 
@@ -1060,6 +1178,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		{
 			error_printf("File is empty!\n");
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 
@@ -1067,11 +1186,16 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		{
 			error_printf("File is too large!\n");
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 				
 		basist::basisu_transcoder dec(&sel_codebook);
 
+		if (pGlobal_codebook_data)
+		{
+			dec.set_global_codebooks(&pGlobal_codebook_data->m_transcoder.get_lowlevel_etc1s_decoder());
+		}
 		if (!opts.m_fuzz_testing)
 		{
 			// Skip the full validation, which CRC16's the entire file.
@@ -1081,6 +1205,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 			{
 				error_printf("File version is unsupported, or file fail CRC checks!\n");
 				if (pCSV_file) fclose(pCSV_file);
+				delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 				return false;
 			}
 		}
@@ -1092,6 +1217,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		{
 			error_printf("Failed retrieving Basis file information!\n");
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 				
@@ -1129,6 +1255,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 			{
 				error_printf("get_image_info() failed!\n");
 				if (pCSV_file) fclose(pCSV_file);
+				delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 				return false;
 			}
 
@@ -1178,6 +1305,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		{
 			error_printf("start_transcoding() failed!\n");
 			if (pCSV_file) fclose(pCSV_file);
+			delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 			return false;
 		}
 
@@ -1260,6 +1388,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed retrieving image level information (%u %u)!\n", image_index, level_index);
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 										
@@ -1292,6 +1421,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, format_iter);
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					
@@ -1336,6 +1466,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 						{
 							error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str());
 							if (pCSV_file) fclose(pCSV_file);
+							delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 							return false;
 						}
 						printf("Wrote KTX file \"%s\"\n", ktx_filename.c_str());
@@ -1364,6 +1495,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 						{
 							error_printf("Failed writing KTX file \"%s\"!\n", ktx_filename.c_str());
 							if (pCSV_file) fclose(pCSV_file);
+							delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 							return false;
 						}
 						printf("Wrote KTX file \"%s\"\n", ktx_filename.c_str());
@@ -1377,6 +1509,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 						{
 							error_printf("Failed retrieving image level information (%u %u)!\n", image_index, level_index);
 							if (pCSV_file) fclose(pCSV_file);
+							delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 							return false;
 						}
 
@@ -1396,6 +1529,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 						if (!save_png(rgb_filename, u, cImageSaveIgnoreAlpha))
 						{
 							error_printf("Failed writing to PNG file \"%s\"\n", rgb_filename.c_str());
+							delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 							return false;
 						}
 						printf("Wrote PNG file \"%s\"\n", rgb_filename.c_str());
@@ -1411,6 +1545,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 							{
 								error_printf("Failed writing to OUT file \"%s\"\n", out_filename.c_str());
 								if (pCSV_file) fclose(pCSV_file);
+								delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 								return false;
 							}
 							printf("Wrote .OUT file \"%s\"\n", out_filename.c_str());
@@ -1427,6 +1562,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 							{
 								error_printf("Failed writing to PNG file \"%s\"\n", a_filename.c_str());
 								if (pCSV_file) fclose(pCSV_file);
+								delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 								return false;
 							}
 							printf("Wrote PNG file \"%s\"\n", a_filename.c_str());
@@ -1440,6 +1576,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 							{
 								error_printf("Failed writing to PNG file \"%s\"\n", rgba_filename.c_str());
 								if (pCSV_file) fclose(pCSV_file);
+								delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 								return false;
 							}
 							printf("Wrote PNG file \"%s\"\n", rgba_filename.c_str());
@@ -1466,6 +1603,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed retrieving image level information (%u %u)!\n", image_index, level_index);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1479,6 +1617,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1495,6 +1634,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed writing to PNG file \"%s\"\n", rgb_filename.c_str());
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					printf("Wrote PNG file \"%s\"\n", rgb_filename.c_str());
@@ -1504,6 +1644,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed writing to PNG file \"%s\"\n", a_filename.c_str());
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					printf("Wrote PNG file \"%s\"\n", a_filename.c_str());
@@ -1525,6 +1666,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed retrieving image level information (%u %u)!\n", image_index, level_index);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1538,6 +1680,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1568,6 +1711,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed writing to PNG file \"%s\"\n", rgb_filename.c_str());
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					printf("Wrote PNG file \"%s\"\n", rgb_filename.c_str());
@@ -1592,6 +1736,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed retrieving image level information (%u %u)!\n", image_index, level_index);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1605,6 +1750,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 				{
 					error_printf("Failed transcoding image level (%u %u %u)!\n", image_index, level_index, transcoder_tex_fmt);
 					if (pCSV_file) fclose(pCSV_file);
+					delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 					return false;
 				}
 
@@ -1636,6 +1782,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed writing to PNG file \"%s\"\n", rgb_filename.c_str());
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					printf("Wrote PNG file \"%s\"\n", rgb_filename.c_str());
@@ -1645,6 +1792,7 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 					{
 						error_printf("Failed writing to PNG file \"%s\"\n", a_filename.c_str());
 						if (pCSV_file) fclose(pCSV_file);
+						delete pGlobal_codebook_data; pGlobal_codebook_data = nullptr;
 						return false;
 					}
 					printf("Wrote PNG file \"%s\"\n", a_filename.c_str());
@@ -1695,6 +1843,8 @@ static bool unpack_and_validate_mode(command_line_params &opts)
 		fclose(pCSV_file);
 		pCSV_file = nullptr;
 	}
+	delete pGlobal_codebook_data; 
+	pGlobal_codebook_data = nullptr;
 
 	return true;
 }

encoder/basisu_backend.cpp

@@ -183,8 +183,16 @@ namespace basisu
 		basisu_frontend& r = *m_pFront_end;
 		//const bool is_video = r.get_params().m_tex_type == basist::cBASISTexTypeVideoFrames;
 
-		//if ((total_block_endpoints_remapped) && (m_params.m_compression_level > 0))
-		if ((total_block_endpoints_remapped) && (m_params.m_compression_level > 1))
+		if (m_params.m_used_global_codebooks)
+		{
+			m_endpoint_remap_table_old_to_new.resize(r.get_total_endpoint_clusters());
+			for (uint32_t i = 0; i < r.get_total_endpoint_clusters(); i++)
+				m_endpoint_remap_table_old_to_new[i] = i;
+		}
+		else
+		{
+			//if ((total_block_endpoints_remapped) && (m_params.m_compression_level > 0))
+			if ((total_block_endpoints_remapped) && (m_params.m_compression_level > 1))
 		{
 			// We've changed the block endpoint indices, so we need to go and adjust the endpoint codebook (remove unused entries, optimize existing entries that have changed)
 			uint_vec new_block_endpoints(get_total_blocks());
@@ -236,6 +244,7 @@ namespace basisu
 		palette_index_reorderer reorderer;
 		reorderer.init((uint32_t)all_endpoint_indices.size(), &all_endpoint_indices[0], r.get_total_endpoint_clusters(), nullptr, nullptr, 0);
 		m_endpoint_remap_table_old_to_new = reorderer.get_remap_table();
+		}
 
 		m_endpoint_remap_table_new_to_old.resize(r.get_total_endpoint_clusters());
 		for (uint32_t i = 0; i < m_endpoint_remap_table_old_to_new.size(); i++)
@@ -248,7 +257,7 @@ namespace basisu
 
 		m_selector_remap_table_new_to_old.resize(r.get_total_selector_clusters());
 
-		if (m_params.m_compression_level == 0)
+		if ((m_params.m_compression_level == 0) || (m_params.m_used_global_codebooks))
 		{
 			for (uint32_t i = 0; i < r.get_total_selector_clusters(); i++)
 				m_selector_remap_table_new_to_old[i] = i;
@@ -1115,7 +1124,7 @@ namespace basisu
 			total_used_selector_history_buf, total_used_selector_history_buf * 100.0f / get_total_blocks());
 
 		//if ((total_endpoint_indices_remapped) && (m_params.m_compression_level > 0))
-		if ((total_endpoint_indices_remapped) && (m_params.m_compression_level > 1))
+		if ((total_endpoint_indices_remapped) && (m_params.m_compression_level > 1) && (!m_params.m_used_global_codebooks))
 		{
 			int_vec unused;
 			r.reoptimize_remapped_endpoints(block_endpoint_indices, unused, false, &block_selector_indices);
@@ -1676,6 +1685,7 @@ namespace basisu
 		//const bool is_video = m_pFront_end->get_params().m_tex_type == basist::cBASISTexTypeVideoFrames;
 		m_output.m_slice_desc = m_slices;
 		m_output.m_etc1s = m_params.m_etc1s;
+		m_output.m_uses_global_codebooks = m_params.m_used_global_codebooks;
 
 		create_endpoint_palette();
 		create_selector_palette();

encoder/basisu_backend.h

@@ -84,6 +84,7 @@ namespace basisu
 		uint32_t m_global_sel_codebook_mod_bits;
 		bool m_use_hybrid_sel_codebooks;
 
+		bool m_used_global_codebooks;
 		basisu_backend_params()
 		{
 			clear();
@@ -102,6 +103,7 @@ namespace basisu
 			m_global_sel_codebook_pal_bits = ETC1_GLOBAL_SELECTOR_CODEBOOK_MAX_PAL_BITS;
 			m_global_sel_codebook_mod_bits = basist::etc1_global_palette_entry_modifier::cTotalBits;
 			m_use_hybrid_sel_codebooks = false;
+			m_used_global_codebooks = false;
 		}
 	};
 
@@ -142,6 +144,7 @@ namespace basisu
 		basist::basis_tex_format m_tex_format;
 
 		bool m_etc1s;
+		bool m_uses_global_codebooks;
 
 		uint32_t m_num_endpoints;
 		uint32_t m_num_selectors;
@@ -164,6 +167,7 @@ namespace basisu
 		{
 			m_tex_format = basist::basis_tex_format::cETC1S;
 			m_etc1s = false;
+			m_uses_global_codebooks = false;
 
 			m_num_endpoints = 0;
 			m_num_selectors = 0;
@@ -201,6 +205,7 @@ namespace basisu
 		uint32_t encode();
 
 		const basisu_backend_output &get_output() const { return m_output; }
+		const basisu_backend_params& get_params() const { return m_params; }
 
 	private:
 		basisu_frontend *m_pFront_end;

encoder/basisu_basis_file.cpp

@@ -47,6 +47,8 @@ namespace basisu
 
 		if (y_flipped)
 			m_header.m_flags = m_header.m_flags | basist::cBASISHeaderFlagYFlipped;
+		if (encoder_output.m_uses_global_codebooks)
+			m_header.m_flags = m_header.m_flags | basist::cBASISHeaderFlagUsesGlobalCodebook;
 				
 		for (uint32_t i = 0; i < encoder_output.m_slice_desc.size(); i++)
 		{
@@ -64,12 +66,26 @@ namespace basisu
 		m_header.m_userdata1 = userdata1;
 
 		m_header.m_total_endpoints = encoder_output.m_num_endpoints;
+		if (!encoder_output.m_uses_global_codebooks)
+		{
 		m_header.m_endpoint_cb_file_ofs = m_endpoint_cb_file_ofs;
 		m_header.m_endpoint_cb_file_size = (uint32_t)encoder_output.m_endpoint_palette.size();
+		}
+		else
+		{
+			assert(!m_endpoint_cb_file_ofs);
+		}
 
 		m_header.m_total_selectors = encoder_output.m_num_selectors;
+		if (!encoder_output.m_uses_global_codebooks)
+		{
 		m_header.m_selector_cb_file_ofs = m_selector_cb_file_ofs;
 		m_header.m_selector_cb_file_size = (uint32_t)encoder_output.m_selector_palette.size();
+		}
+		else
+		{
+			assert(!m_selector_cb_file_ofs);
+		}
 
 		m_header.m_tables_file_ofs = m_tables_file_ofs;
 		m_header.m_tables_file_size = (uint32_t)encoder_output.m_slice_image_tables.size();
@@ -134,6 +150,8 @@ namespace basisu
 		assert(m_comp_data.size() == m_slice_descs_file_ofs);
 		append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&m_images_descs[0]), m_images_descs.size() * sizeof(m_images_descs[0]));
 
+		if (!encoder_output.m_uses_global_codebooks)
+		{
 		if (encoder_output.m_endpoint_palette.size())
 		{
 			assert(m_comp_data.size() == m_endpoint_cb_file_ofs);
@@ -144,6 +162,7 @@ namespace basisu
 		{
 			assert(m_comp_data.size() == m_selector_cb_file_ofs);
 			append_vector(m_comp_data, reinterpret_cast<const uint8_t*>(&encoder_output.m_selector_palette[0]), encoder_output.m_selector_palette.size());
+			}
 		}
 
 		if (encoder_output.m_slice_image_tables.size())
@@ -179,8 +198,17 @@ namespace basisu
 		const basisu_backend_slice_desc_vec &slice_descs = encoder_output.m_slice_desc;
 
 		// The Basis file uses 32-bit fields for lots of stuff, so make sure it's not too large.
-		uint64_t check_size = (uint64_t)sizeof(basist::basis_file_header) + (uint64_t)sizeof(basist::basis_slice_desc) * slice_descs.size() + 
+		uint64_t check_size = 0;
+		if (!encoder_output.m_uses_global_codebooks)
+		{
+			check_size = (uint64_t)sizeof(basist::basis_file_header) + (uint64_t)sizeof(basist::basis_slice_desc) * slice_descs.size() +
 			(uint64_t)encoder_output.m_endpoint_palette.size() + (uint64_t)encoder_output.m_selector_palette.size() + (uint64_t)encoder_output.m_slice_image_tables.size();
+		}
+		else
+		{
+			check_size = (uint64_t)sizeof(basist::basis_file_header) + (uint64_t)sizeof(basist::basis_slice_desc) * slice_descs.size() +
+				(uint64_t)encoder_output.m_slice_image_tables.size();
+		}
 		if (check_size >= 0xFFFF0000ULL)
 		{
 			error_printf("basisu_file::init: File is too large!\n");
@@ -191,9 +219,18 @@ namespace basisu
 		m_slice_descs_file_ofs = sizeof(basist::basis_file_header);
 		if (encoder_output.m_tex_format == basist::basis_tex_format::cETC1S)
 		{
+			if (encoder_output.m_uses_global_codebooks)
+			{
+				m_endpoint_cb_file_ofs = 0;
+				m_selector_cb_file_ofs = 0;
+				m_tables_file_ofs = m_slice_descs_file_ofs + sizeof(basist::basis_slice_desc) * (uint32_t)slice_descs.size();
+			}
+			else
+			{
 			m_endpoint_cb_file_ofs = m_slice_descs_file_ofs + sizeof(basist::basis_slice_desc) * (uint32_t)slice_descs.size();
 			m_selector_cb_file_ofs = m_endpoint_cb_file_ofs + (uint32_t)encoder_output.m_endpoint_palette.size();
 			m_tables_file_ofs = m_selector_cb_file_ofs + (uint32_t)encoder_output.m_selector_palette.size();
+			}
 			m_first_image_file_ofs = m_tables_file_ofs + (uint32_t)encoder_output.m_slice_image_tables.size();
 		}
 		else

encoder/basisu_comp.cpp

@@ -61,6 +61,7 @@ namespace basisu
 			PRINT_BOOL_VALUE(m_uastc);
 			PRINT_BOOL_VALUE(m_y_flip);
 			PRINT_BOOL_VALUE(m_debug);
+			PRINT_BOOL_VALUE(m_validate);
 			PRINT_BOOL_VALUE(m_debug_images);
 			PRINT_BOOL_VALUE(m_global_sel_pal);
 			PRINT_BOOL_VALUE(m_auto_global_sel_pal);
@@ -121,6 +122,11 @@ namespace basisu
 			PRINT_BOOL_VALUE(m_rdo_uastc_multithreading);
 
 			PRINT_FLOAT_VALUE(m_resample_factor);
+			printf("Has global codebooks: %u\n", m_params.m_pGlobal_codebooks ? 1 : 0);
+			if (m_params.m_pGlobal_codebooks)
+			{
+				printf("Global codebook endpoints: %u selectors: %u\n", m_params.m_pGlobal_codebooks->get_endpoints().size(), m_params.m_pGlobal_codebooks->get_selectors().size());
+			}
 						
 #undef PRINT_BOOL_VALUE
 #undef PRINT_INT_VALUE
@@ -1006,7 +1012,9 @@ namespace basisu
 		p.m_tex_type = m_params.m_tex_type;
 		p.m_multithreaded = m_params.m_multithreading;
 		p.m_disable_hierarchical_endpoint_codebooks = m_params.m_disable_hierarchical_endpoint_codebooks;
+		p.m_validate = m_params.m_validate;
 		p.m_pJob_pool = m_params.m_pJob_pool;
+		p.m_pGlobal_codebooks = m_params.m_pGlobal_codebooks;
 
 		if ((m_params.m_global_sel_pal) || (m_auto_global_sel_pal))
 		{
@@ -1113,6 +1121,7 @@ namespace basisu
 		backend_params.m_global_sel_codebook_pal_bits = m_frontend.get_params().m_num_global_sel_codebook_pal_bits;
 		backend_params.m_global_sel_codebook_mod_bits = m_frontend.get_params().m_num_global_sel_codebook_mod_bits;
 		backend_params.m_use_hybrid_sel_codebooks = m_frontend.get_params().m_use_hybrid_selector_codebooks;
+		backend_params.m_used_global_codebooks = m_frontend.get_params().m_pGlobal_codebooks != nullptr;
 
 		m_backend.init(&m_frontend, backend_params, m_slice_descs, m_params.m_pSel_codebook);
 		uint32_t total_packed_bytes = m_backend.encode();
@@ -1166,6 +1175,10 @@ namespace basisu
 		m_decoded_output_textures_unpacked_bc7.resize(m_slice_descs.size());
 								
 		tm.start();
+		if (m_params.m_pGlobal_codebooks)
+		{
+			decoder.set_global_codebooks(m_params.m_pGlobal_codebooks);
+		}
 
 		if (!decoder.start_transcoding(&comp_data[0], (uint32_t)comp_data.size()))
 		{

encoder/basisu_comp.h

@@ -230,6 +230,7 @@ namespace basisu
 
 			m_y_flip.clear();
 			m_debug.clear();
+			m_validate.clear();
 			m_debug_images.clear();
 			m_global_sel_pal.clear();
 			m_auto_global_sel_pal.clear();
@@ -289,6 +290,7 @@ namespace basisu
 
 			m_resample_factor.clear();
 
+			m_pGlobal_codebooks = nullptr;
 			m_pJob_pool = nullptr;
 		}
 				
@@ -319,6 +321,7 @@ namespace basisu
 		
 		// Output debug information during compression
 		bool_param<false> m_debug;
+		bool_param<false> m_validate;
 		
 		// m_debug_images is pretty slow
 		bool_param<false> m_debug_images;
@@ -407,6 +410,7 @@ namespace basisu
 		bool_param<true> m_rdo_uastc_multithreading;
 
 		param<float> m_resample_factor;
+		const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks;
 
 		job_pool *m_pJob_pool;
 	};

encoder/basisu_frontend.cpp

@@ -196,107 +196,114 @@ namespace basisu
 
 		init_etc1_images();
 
-		init_endpoint_training_vectors();
-
-		generate_endpoint_clusters();
-				
-		for (uint32_t refine_endpoint_step = 0; refine_endpoint_step < m_num_endpoint_codebook_iterations; refine_endpoint_step++)
+		if (m_params.m_pGlobal_codebooks)
 		{
-			BASISU_FRONTEND_VERIFY(check_etc1s_constraints());
+			init_global_codebooks();
+		}
+		else
+		{
+			init_endpoint_training_vectors();
 
-			if (refine_endpoint_step)
+			generate_endpoint_clusters();
+				
+			for (uint32_t refine_endpoint_step = 0; refine_endpoint_step < m_num_endpoint_codebook_iterations; refine_endpoint_step++)
 			{
-				introduce_new_endpoint_clusters();
-			}
+				BASISU_FRONTEND_VERIFY(check_etc1s_constraints());
 
-			generate_endpoint_codebook(refine_endpoint_step);
-
-			if ((m_params.m_debug_images) && (m_params.m_dump_endpoint_clusterization))
-			{
-				char buf[256];
-				snprintf(buf, sizeof(buf), "endpoint_cluster_vis_pre_%u.png", refine_endpoint_step);
-				dump_endpoint_clusterization_visualization(buf, false);
-			}
-
-			bool early_out = false;
-
-			if (m_endpoint_refinement)
-			{
-				//dump_endpoint_clusterization_visualization("endpoint_clusters_before_refinement.png");
-
-				if (!refine_endpoint_clusterization())
-					early_out = true;
-
-				if ((m_params.m_tex_type == basist::cBASISTexTypeVideoFrames) && (!refine_endpoint_step) && (m_num_endpoint_codebook_iterations == 1))
+				if (refine_endpoint_step)
 				{
-					eliminate_redundant_or_empty_endpoint_clusters();
-					generate_endpoint_codebook(refine_endpoint_step);
+					introduce_new_endpoint_clusters();
 				}
 
+				generate_endpoint_codebook(refine_endpoint_step);
+
 				if ((m_params.m_debug_images) && (m_params.m_dump_endpoint_clusterization))
 				{
 					char buf[256];
-					snprintf(buf, sizeof(buf), "endpoint_cluster_vis_post_%u.png", refine_endpoint_step);
-
+					snprintf(buf, sizeof(buf), "endpoint_cluster_vis_pre_%u.png", refine_endpoint_step);
 					dump_endpoint_clusterization_visualization(buf, false);
-					snprintf(buf, sizeof(buf), "endpoint_cluster_colors_vis_post_%u.png", refine_endpoint_step);
-
-					dump_endpoint_clusterization_visualization(buf, true);
 				}
-			}
+
+				bool early_out = false;
+
+				if (m_endpoint_refinement)
+				{
+					//dump_endpoint_clusterization_visualization("endpoint_clusters_before_refinement.png");
+
+					if (!refine_endpoint_clusterization())
+						early_out = true;
+
+					if ((m_params.m_tex_type == basist::cBASISTexTypeVideoFrames) && (!refine_endpoint_step) && (m_num_endpoint_codebook_iterations == 1))
+					{
+						eliminate_redundant_or_empty_endpoint_clusters();
+						generate_endpoint_codebook(refine_endpoint_step);
+					}
+
+					if ((m_params.m_debug_images) && (m_params.m_dump_endpoint_clusterization))
+					{
+						char buf[256];
+						snprintf(buf, sizeof(buf), "endpoint_cluster_vis_post_%u.png", refine_endpoint_step);
+
+						dump_endpoint_clusterization_visualization(buf, false);
+						snprintf(buf, sizeof(buf), "endpoint_cluster_colors_vis_post_%u.png", refine_endpoint_step);
+
+						dump_endpoint_clusterization_visualization(buf, true);
+					}
+				}
 						
-			eliminate_redundant_or_empty_endpoint_clusters();
+				eliminate_redundant_or_empty_endpoint_clusters();
 
-			if (m_params.m_debug_stats)
-				debug_printf("Total endpoint clusters: %u\n", (uint32_t)m_endpoint_clusters.size());
+				if (m_params.m_debug_stats)
+					debug_printf("Total endpoint clusters: %u\n", (uint32_t)m_endpoint_clusters.size());
 
-			if (early_out)
-				break;
-		}
+				if (early_out)
+					break;
+			}
 
-		BASISU_FRONTEND_VERIFY(check_etc1s_constraints());
+			BASISU_FRONTEND_VERIFY(check_etc1s_constraints());
 
-		generate_block_endpoint_clusters();
+			generate_block_endpoint_clusters();
 
-		create_initial_packed_texture();
+			create_initial_packed_texture();
 
-		generate_selector_clusters();
+			generate_selector_clusters();
 
-		if (m_use_hierarchical_selector_codebooks)
-			compute_selector_clusters_within_each_parent_cluster();
+			if (m_use_hierarchical_selector_codebooks)
+				compute_selector_clusters_within_each_parent_cluster();
 				
-		if (m_params.m_compression_level == 0)
-		{
-			create_optimized_selector_codebook(0);
-
-			find_optimal_selector_clusters_for_each_block();
-			
-			introduce_special_selector_clusters();
-		}
-		else
-		{
-			const uint32_t num_refine_selector_steps = m_params.m_pGlobal_sel_codebook ? 1 : m_num_selector_codebook_iterations;
-			for (uint32_t refine_selector_steps = 0; refine_selector_steps < num_refine_selector_steps; refine_selector_steps++)
+			if (m_params.m_compression_level == 0)
 			{
-				create_optimized_selector_codebook(refine_selector_steps);
+				create_optimized_selector_codebook(0);
 
 				find_optimal_selector_clusters_for_each_block();
-
+			
 				introduce_special_selector_clusters();
-				
-				if ((m_params.m_compression_level >= 4) || (m_params.m_tex_type == basist::cBASISTexTypeVideoFrames))
+			}
+			else
+			{
+				const uint32_t num_refine_selector_steps = m_params.m_pGlobal_sel_codebook ? 1 : m_num_selector_codebook_iterations;
+				for (uint32_t refine_selector_steps = 0; refine_selector_steps < num_refine_selector_steps; refine_selector_steps++)
 				{
-					if (!refine_block_endpoints_given_selectors())
-						break;
+					create_optimized_selector_codebook(refine_selector_steps);
+
+					find_optimal_selector_clusters_for_each_block();
+
+					introduce_special_selector_clusters();
+				
+					if ((m_params.m_compression_level >= 4) || (m_params.m_tex_type == basist::cBASISTexTypeVideoFrames))
+					{
+						if (!refine_block_endpoints_given_selectors())
+							break;
+					}
 				}
 			}
+
+			optimize_selector_codebook();
+
+			if (m_params.m_debug_stats)
+				debug_printf("Total selector clusters: %u\n", (uint32_t)m_selector_cluster_block_indices.size());
 		}
 
-		optimize_selector_codebook();
-
-		if (m_params.m_debug_stats)
-			debug_printf("Total selector clusters: %u\n", (uint32_t)m_selector_cluster_block_indices.size());
-
 		finalize();
 
 		if (m_params.m_validate)
@@ -310,6 +317,258 @@ namespace basisu
 		return true;
 	}
 
+	bool basisu_frontend::init_global_codebooks()
+	{
+		const basist::basisu_lowlevel_etc1s_transcoder* pTranscoder = m_params.m_pGlobal_codebooks;
+
+		const basist::basisu_lowlevel_etc1s_transcoder::endpoint_vec& endpoints = pTranscoder->get_endpoints();
+		const basist::basisu_lowlevel_etc1s_transcoder::selector_vec& selectors = pTranscoder->get_selectors();
+				
+		m_endpoint_cluster_etc_params.resize(endpoints.size());
+		for (uint32_t i = 0; i < endpoints.size(); i++)
+		{
+			m_endpoint_cluster_etc_params[i].m_inten_table[0] = endpoints[i].m_inten5;
+			m_endpoint_cluster_etc_params[i].m_inten_table[1] = endpoints[i].m_inten5;
+
+			m_endpoint_cluster_etc_params[i].m_color_unscaled[0].set(endpoints[i].m_color5.r, endpoints[i].m_color5.g, endpoints[i].m_color5.b, 255);
+			m_endpoint_cluster_etc_params[i].m_color_used[0] = true;
+			m_endpoint_cluster_etc_params[i].m_valid = true;
+		}
+
+		m_optimized_cluster_selectors.resize(selectors.size());
+		for (uint32_t i = 0; i < m_optimized_cluster_selectors.size(); i++)
+		{
+			for (uint32_t y = 0; y < 4; y++)
+				for (uint32_t x = 0; x < 4; x++)
+					m_optimized_cluster_selectors[i].set_selector(x, y, selectors[i].get_selector(x, y));
+		}
+
+		m_block_endpoint_clusters_indices.resize(m_total_blocks);
+
+		m_orig_encoded_blocks.resize(m_total_blocks);
+
+		m_block_selector_cluster_index.resize(m_total_blocks);
+
+#if 0
+		for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N)
+		{
+			const uint32_t first_index = block_index_iter;
+			const uint32_t last_index = minimum<uint32_t>(m_total_blocks, first_index + N);
+
+#ifndef __EMSCRIPTEN__
+			m_params.m_pJob_pool->add_job([this, first_index, last_index] {
+#endif
+
+				for (uint32_t block_index = first_index; block_index < last_index; block_index++)
+				{
+					const etc_block& blk = m_etc1_blocks_etc1s[block_index];
+
+					const uint32_t block_endpoint_index = m_block_endpoint_clusters_indices[block_index][0];
+
+					etc_block trial_blk;
+					trial_blk.set_block_color5_etc1s(blk.m_color_unscaled[0]);
+					trial_blk.set_flip_bit(true);
+
+					uint64_t best_err = UINT64_MAX;
+					uint32_t best_index = 0;
+
+					for (uint32_t i = 0; i < m_optimized_cluster_selectors.size(); i++)
+					{
+						trial_blk.set_raw_selector_bits(m_optimized_cluster_selectors[i].get_raw_selector_bits());
+
+						const uint64_t cur_err = trial_blk.evaluate_etc1_error(get_source_pixel_block(block_index).get_ptr(), m_params.m_perceptual);
+						if (cur_err < best_err)
+						{
+							best_err = cur_err;
+							best_index = i;
+							if (!cur_err)
+								break;
+						}
+
+					} // block_index
+
+					m_block_selector_cluster_index[block_index] = best_index;
+				}
+
+#ifndef __EMSCRIPTEN__
+				});
+#endif
+
+		}
+
+#ifndef __EMSCRIPTEN__
+		m_params.m_pJob_pool->wait_for_all();
+#endif
+
+		m_encoded_blocks.resize(m_total_blocks);
+		for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++)
+		{
+			const uint32_t endpoint_index = m_block_endpoint_clusters_indices[block_index][0];
+			const uint32_t selector_index = m_block_selector_cluster_index[block_index];
+
+			etc_block& blk = m_encoded_blocks[block_index];
+
+			blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[endpoint_index].m_color_unscaled[0]);
+			blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[endpoint_index].m_inten_table[0]);
+			blk.set_flip_bit(true);
+			blk.set_raw_selector_bits(m_optimized_cluster_selectors[selector_index].get_raw_selector_bits());
+		}
+#endif
+
+		const uint32_t NUM_PASSES = 3;
+		for (uint32_t pass = 0; pass < NUM_PASSES; pass++)
+		{
+			debug_printf("init_global_codebooks: pass %u\n", pass);
+
+			const uint32_t N = 128;
+			for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N)
+			{
+				const uint32_t first_index = block_index_iter;
+				const uint32_t last_index = minimum<uint32_t>(m_total_blocks, first_index + N);
+
+#ifndef __EMSCRIPTEN__
+				m_params.m_pJob_pool->add_job([this, first_index, last_index, pass] {
+#endif
+										
+					for (uint32_t block_index = first_index; block_index < last_index; block_index++)
+					{
+						const etc_block& blk = pass ? m_encoded_blocks[block_index] : m_etc1_blocks_etc1s[block_index];
+						const uint32_t blk_raw_selector_bits = blk.get_raw_selector_bits();
+
+						etc_block trial_blk(blk);
+						trial_blk.set_raw_selector_bits(blk_raw_selector_bits);
+						trial_blk.set_flip_bit(true);
+
+						uint64_t best_err = UINT64_MAX;
+						uint32_t best_index = 0;
+						etc_block best_block(trial_blk);
+												
+						for (uint32_t i = 0; i < m_endpoint_cluster_etc_params.size(); i++)
+						{
+							if (m_endpoint_cluster_etc_params[i].m_inten_table[0] > blk.get_inten_table(0))
+								continue;
+
+							trial_blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[i].m_color_unscaled[0]);
+							trial_blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[i].m_inten_table[0]);
+
+							const color_rgba* pSource_pixels = get_source_pixel_block(block_index).get_ptr();
+							uint64_t cur_err;
+							if (!pass)
+								cur_err = trial_blk.determine_selectors(pSource_pixels, m_params.m_perceptual);
+							else
+								cur_err = trial_blk.evaluate_etc1_error(pSource_pixels, m_params.m_perceptual);
+
+							if (cur_err < best_err)
+							{
+								best_err = cur_err;
+								best_index = i;
+								best_block = trial_blk;
+
+								if (!cur_err)
+									break;
+							}
+						}
+
+						m_block_endpoint_clusters_indices[block_index][0] = best_index;
+						m_block_endpoint_clusters_indices[block_index][1] = best_index;
+
+						m_orig_encoded_blocks[block_index] = best_block;
+
+					} // block_index
+
+#ifndef __EMSCRIPTEN__
+					});
+#endif
+
+			}
+
+#ifndef __EMSCRIPTEN__
+			m_params.m_pJob_pool->wait_for_all();
+#endif
+
+			m_endpoint_clusters.resize(0);
+			m_endpoint_clusters.resize(endpoints.size());
+			for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++)
+			{
+				const uint32_t endpoint_cluster_index = m_block_endpoint_clusters_indices[block_index][0];
+				m_endpoint_clusters[endpoint_cluster_index].push_back(block_index * 2);
+				m_endpoint_clusters[endpoint_cluster_index].push_back(block_index * 2 + 1);
+			}
+
+			m_block_selector_cluster_index.resize(m_total_blocks);
+
+			for (uint32_t block_index_iter = 0; block_index_iter < m_total_blocks; block_index_iter += N)
+			{
+				const uint32_t first_index = block_index_iter;
+				const uint32_t last_index = minimum<uint32_t>(m_total_blocks, first_index + N);
+
+#ifndef __EMSCRIPTEN__
+				m_params.m_pJob_pool->add_job([this, first_index, last_index, pass] {
+#endif
+
+					for (uint32_t block_index = first_index; block_index < last_index; block_index++)
+					{
+						const uint32_t block_endpoint_index = m_block_endpoint_clusters_indices[block_index][0];
+
+						etc_block trial_blk;
+						trial_blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_color_unscaled[0]);
+						trial_blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[block_endpoint_index].m_inten_table[0]);
+						trial_blk.set_flip_bit(true);
+
+						uint64_t best_err = UINT64_MAX;
+						uint32_t best_index = 0;
+
+						for (uint32_t i = 0; i < m_optimized_cluster_selectors.size(); i++)
+						{
+							trial_blk.set_raw_selector_bits(m_optimized_cluster_selectors[i].get_raw_selector_bits());
+
+							const uint64_t cur_err = trial_blk.evaluate_etc1_error(get_source_pixel_block(block_index).get_ptr(), m_params.m_perceptual);
+							if (cur_err < best_err)
+							{
+								best_err = cur_err;
+								best_index = i;
+								if (!cur_err)
+									break;
+							}
+
+						} // block_index
+
+						m_block_selector_cluster_index[block_index] = best_index;
+					}
+
+#ifndef __EMSCRIPTEN__
+					});
+#endif
+
+			}
+
+#ifndef __EMSCRIPTEN__
+			m_params.m_pJob_pool->wait_for_all();
+#endif
+
+			m_encoded_blocks.resize(m_total_blocks);
+			for (uint32_t block_index = 0; block_index < m_total_blocks; block_index++)
+			{
+				const uint32_t endpoint_index = m_block_endpoint_clusters_indices[block_index][0];
+				const uint32_t selector_index = m_block_selector_cluster_index[block_index];
+
+				etc_block& blk = m_encoded_blocks[block_index];
+
+				blk.set_block_color5_etc1s(m_endpoint_cluster_etc_params[endpoint_index].m_color_unscaled[0]);
+				blk.set_inten_tables_etc1s(m_endpoint_cluster_etc_params[endpoint_index].m_inten_table[0]);
+				blk.set_flip_bit(true);
+				blk.set_raw_selector_bits(m_optimized_cluster_selectors[selector_index].get_raw_selector_bits());
+			}
+
+		} // pass
+
+		m_selector_cluster_block_indices.resize(selectors.size());
+		for (uint32_t block_index = 0; block_index < m_etc1_blocks_etc1s.size(); block_index++)
+			m_selector_cluster_block_indices[m_block_selector_cluster_index[block_index]].push_back(block_index);
+				
+		return true;
+	}
+
 	void basisu_frontend::introduce_special_selector_clusters()
 	{
 		debug_printf("introduce_special_selector_clusters\n");

encoder/basisu_frontend.h

@@ -18,6 +18,7 @@
 #include "basisu_gpu_texture.h"
 #include "basisu_global_selector_palette_helpers.h"
 #include "../transcoder/basisu_file_headers.h"
+#include "../transcoder/basisu_transcoder.h"
 
 namespace basisu
 {
@@ -83,6 +84,7 @@ namespace basisu
 				m_use_hybrid_selector_codebooks(false),
 				m_hybrid_codebook_quality_thresh(0.0f),
 				m_tex_type(basist::cBASISTexType2D),
+				m_pGlobal_codebooks(nullptr),
 				
 				m_pJob_pool(nullptr)
 			{
@@ -110,6 +112,7 @@ namespace basisu
 			bool m_use_hybrid_selector_codebooks;
 			float m_hybrid_codebook_quality_thresh;
 			basist::basis_texture_type m_tex_type;
+			const basist::basisu_lowlevel_etc1s_transcoder *m_pGlobal_codebooks;
 			
 			job_pool *m_pJob_pool;
 		};
@@ -330,6 +333,7 @@ namespace basisu
 		//-----------------------------------------------------------------------------
 
 		void init_etc1_images();
+		bool init_global_codebooks();
 		void init_endpoint_training_vectors();
 		void dump_endpoint_clusterization_visualization(const char *pFilename, bool vis_endpoint_colors);
 		void generate_endpoint_clusters();

transcoder/basisu_file_headers.h

@@ -49,7 +49,8 @@ namespace basist
 	{
 		cBASISHeaderFlagETC1S = 1,					// Always set for ETC1S files. Not set for UASTC files.
 		cBASISHeaderFlagYFlipped = 2,				// Set if the texture had to be Y flipped before encoding
-		cBASISHeaderFlagHasAlphaSlices = 4		// True if any slices contain alpha (for ETC1S, if the odd slices contain alpha data)
+		cBASISHeaderFlagHasAlphaSlices = 4,		// True if any slices contain alpha (for ETC1S, if the odd slices contain alpha data)
+		cBASISHeaderFlagUsesGlobalCodebook = 8 // For ETC1S files, this will be true if the file utilizes a codebook from another .basis file. 
 	};
 
 	// The image type field attempts to describe how to interpret the image data in a Basis file.

transcoder/basisu_transcoder.cpp

@@ -7515,6 +7515,7 @@ namespace basist
 #endif // BASISD_SUPPORT_PVRTC2
 
 	basisu_lowlevel_etc1s_transcoder::basisu_lowlevel_etc1s_transcoder(const etc1_global_selector_codebook* pGlobal_sel_codebook) :
+		m_pGlobal_codebook(nullptr),
 		m_pGlobal_sel_codebook(pGlobal_sel_codebook),
 		m_selector_history_buf_size(0)
 	{
@@ -7524,6 +7525,11 @@ namespace basist
 		uint32_t num_endpoints, const uint8_t* pEndpoints_data, uint32_t endpoints_data_size,
 		uint32_t num_selectors, const uint8_t* pSelectors_data, uint32_t selectors_data_size)
 	{
+		if (m_pGlobal_codebook)
+		{
+			BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::decode_palettes: fail 11\n");
+			return false;
+		}
 		bitwise_decoder sym_codec;
 
 		huffman_decoding_table color5_delta_model0, color5_delta_model1, color5_delta_model2, inten_delta_model;
@@ -7566,7 +7572,7 @@ namespace basist
 
 		const bool endpoints_are_grayscale = sym_codec.get_bits(1) != 0;
 
-		m_endpoints.resize(num_endpoints);
+		m_local_endpoints.resize(num_endpoints);
 
 		color32 prev_color5(16, 16, 16, 0);
 		uint32_t prev_inten = 0;
@@ -7574,8 +7580,8 @@ namespace basist
 		for (uint32_t i = 0; i < num_endpoints; i++)
 		{
 			uint32_t inten_delta = sym_codec.decode_huffman(inten_delta_model);
-			m_endpoints[i].m_inten5 = static_cast<uint8_t>((inten_delta + prev_inten) & 7);
-			prev_inten = m_endpoints[i].m_inten5;
+			m_local_endpoints[i].m_inten5 = static_cast<uint8_t>((inten_delta + prev_inten) & 7);
+			prev_inten = m_local_endpoints[i].m_inten5;
 
 			for (uint32_t c = 0; c < (endpoints_are_grayscale ? 1U : 3U); c++)
 			{
@@ -7589,21 +7595,21 @@ namespace basist
 
 				int v = (prev_color5[c] + delta) & 31;
 
-				m_endpoints[i].m_color5[c] = static_cast<uint8_t>(v);
+				m_local_endpoints[i].m_color5[c] = static_cast<uint8_t>(v);
 
 				prev_color5[c] = static_cast<uint8_t>(v);
 			}
 
 			if (endpoints_are_grayscale)
 			{
-				m_endpoints[i].m_color5[1] = m_endpoints[i].m_color5[0];
-				m_endpoints[i].m_color5[2] = m_endpoints[i].m_color5[0];
+				m_local_endpoints[i].m_color5[1] = m_local_endpoints[i].m_color5[0];
+				m_local_endpoints[i].m_color5[2] = m_local_endpoints[i].m_color5[0];
 			}
 		}
 
 		sym_codec.stop();
 
-		m_selectors.resize(num_selectors);
+		m_local_selectors.resize(num_selectors);
 		
 		if (!sym_codec.init(pSelectors_data, selectors_data_size))
 		{
@@ -7657,9 +7663,9 @@ namespace basist
 				// TODO: Optimize this
 				for (uint32_t y = 0; y < 4; y++)
 					for (uint32_t x = 0; x < 4; x++)
-						m_selectors[i].set_selector(x, y, e[x + y * 4]);
+						m_local_selectors[i].set_selector(x, y, e[x + y * 4]);
 
-				m_selectors[i].init_flags();
+				m_local_selectors[i].init_flags();
 			}
 		}
 		else
@@ -7729,7 +7735,7 @@ namespace basist
 
 						for (uint32_t y = 0; y < 4; y++)
 							for (uint32_t x = 0; x < 4; x++)
-								m_selectors[q].set_selector(x, y, e[x + y * 4]);
+								m_local_selectors[q].set_selector(x, y, e[x + y * 4]);
 					}
 					else
 					{
@@ -7738,11 +7744,11 @@ namespace basist
 							uint32_t cur_byte = sym_codec.get_bits(8);
 
 							for (uint32_t k = 0; k < 4; k++)
-								m_selectors[q].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
+								m_local_selectors[q].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
 						}
 					}
 
-					m_selectors[q].init_flags();
+					m_local_selectors[q].init_flags();
 				}
 			}
 			else
@@ -7758,10 +7764,10 @@ namespace basist
 							uint32_t cur_byte = sym_codec.get_bits(8);
 
 							for (uint32_t k = 0; k < 4; k++)
-								m_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
+								m_local_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
 						}
 
-						m_selectors[i].init_flags();
+						m_local_selectors[i].init_flags();
 					}
 				}
 				else
@@ -7790,9 +7796,9 @@ namespace basist
 								prev_bytes[j] = static_cast<uint8_t>(cur_byte);
 
 								for (uint32_t k = 0; k < 4; k++)
-									m_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
+									m_local_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
 							}
-							m_selectors[i].init_flags();
+							m_local_selectors[i].init_flags();
 							continue;
 						}
 
@@ -7804,9 +7810,9 @@ namespace basist
 							prev_bytes[j] = static_cast<uint8_t>(cur_byte);
 
 							for (uint32_t k = 0; k < 4; k++)
-								m_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
+								m_local_selectors[i].set_selector(k, j, (cur_byte >> (k * 2)) & 3);
 						}
-						m_selectors[i].init_flags();
+						m_local_selectors[i].init_flags();
 					}
 				}
 			}
@@ -7944,7 +7950,7 @@ namespace basist
 
 		approx_move_to_front selector_history_buf(m_selector_history_buf_size);
 
-		const uint32_t SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX = (uint32_t)m_selectors.size();
+		const uint32_t SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX = (uint32_t)m_local_selectors.size();
 		const uint32_t SELECTOR_HISTORY_BUF_RLE_SYMBOL_INDEX = m_selector_history_buf_size + SELECTOR_HISTORY_BUF_FIRST_SYMBOL_INDEX;
 		uint32_t cur_selector_rle_count = 0;
 
@@ -7977,6 +7983,13 @@ namespace basist
 		int prev_endpoint_pred_sym = 0;
 		int endpoint_pred_repeat_count = 0;
 		uint32_t prev_endpoint_index = 0;
+		const endpoint_vec& endpoints = m_pGlobal_codebook ? m_pGlobal_codebook->m_local_endpoints : m_local_endpoints;
+		const selector_vec& selectors = m_pGlobal_codebook ? m_pGlobal_codebook->m_local_selectors : m_local_selectors;
+		if (!endpoints.size() || !selectors.size())
+		{
+			BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: global codebooks must be unpacked first\n");
+			return false;
+		}
 
 		for (uint32_t block_y = 0; block_y < num_blocks_y; block_y++)
 		{
@@ -8078,8 +8091,8 @@ namespace basist
 					const uint32_t delta_sym = sym_codec.decode_huffman(m_delta_endpoint_model);
 
 					endpoint_index = delta_sym + prev_endpoint_index;
-					if (endpoint_index >= m_endpoints.size())
-						endpoint_index -= (int)m_endpoints.size();
+					if (endpoint_index >= endpoints.size())
+						endpoint_index -= (int)endpoints.size();
 				}
 
 				pState->m_block_endpoint_preds[cur_block_endpoint_pred_array][block_x].m_endpoint_index = (uint16_t)endpoint_index;
@@ -8094,7 +8107,7 @@ namespace basist
 					{
 						cur_selector_rle_count--;
 
-						selector_sym = (int)m_selectors.size();
+						selector_sym = (int)selectors.size();
 					}
 					else
 					{
@@ -8118,17 +8131,17 @@ namespace basist
 								return false;
 							}
 
-							selector_sym = (int)m_selectors.size();
+							selector_sym = (int)selectors.size();
 
 							cur_selector_rle_count--;
 						}
 					}
 
-					if (selector_sym >= (int)m_selectors.size())
+					if (selector_sym >= (int)selectors.size())
 					{
 						assert(m_selector_history_buf_size > 0);
 
-						int history_buf_index = selector_sym - (int)m_selectors.size();
+						int history_buf_index = selector_sym - (int)selectors.size();
 
 						if (history_buf_index >= (int)selector_history_buf.size())
 						{
@@ -8153,7 +8166,7 @@ namespace basist
 					}
 				}
 
-				if ((endpoint_index >= m_endpoints.size()) || (selector_index >= m_selectors.size()))
+				if ((endpoint_index >= endpoints.size()) || (selector_index >= selectors.size()))
 				{
 					// The file is corrupted or we've got a bug.
 					BASISU_DEVEL_ERROR("basisu_lowlevel_etc1s_transcoder::transcode_slice: invalid datastream (5)\n");
@@ -8177,8 +8190,8 @@ namespace basist
 				}
 #endif
 
-				const endpoint* pEndpoints = &m_endpoints[endpoint_index];
-				const selector* pSelector = &m_selectors[selector_index];
+				const endpoint* pEndpoints = &endpoints[endpoint_index];
+				const selector* pSelector = &selectors[selector_index];
 
 				switch (fmt)
 				{
@@ -8282,8 +8295,8 @@ namespace basist
 
 					const uint16_t* pAlpha_block = reinterpret_cast<uint16_t*>(static_cast<uint8_t*>(pAlpha_blocks) + (block_x + block_y * num_blocks_x) * sizeof(uint32_t));
 
-					const endpoint* pAlpha_endpoints = &m_endpoints[pAlpha_block[0]];
-					const selector* pAlpha_selector = &m_selectors[pAlpha_block[1]];
+					const endpoint* pAlpha_endpoints = &endpoints[pAlpha_block[0]];
+					const selector* pAlpha_selector = &selectors[pAlpha_block[1]];
 
 					const color32& alpha_base_color = pAlpha_endpoints->m_color5;
 					const uint32_t alpha_inten_table = pAlpha_endpoints->m_inten5;
@@ -8342,7 +8355,7 @@ namespace basist
 				{
 #if BASISD_SUPPORT_ASTC
 					void* pDst_block = static_cast<uint8_t*>(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes;
-					convert_etc1s_to_astc_4x4(pDst_block, pEndpoints, pSelector, transcode_alpha, &m_endpoints[0], &m_selectors[0]);
+					convert_etc1s_to_astc_4x4(pDst_block, pEndpoints, pSelector, transcode_alpha, &endpoints[0], &selectors[0]);
 #else
 					assert(0);
 #endif
@@ -8388,7 +8401,7 @@ namespace basist
 
 					void* pDst_block = static_cast<uint8_t*>(pDst_blocks) + (block_x + block_y * output_row_pitch_in_blocks_or_pixels) * output_block_or_pixel_stride_in_bytes;
 										
-					convert_etc1s_to_pvrtc2_rgba(pDst_block, pEndpoints, pSelector, &m_endpoints[0], &m_selectors[0]);
+					convert_etc1s_to_pvrtc2_rgba(pDst_block, pEndpoints, pSelector, &endpoints[0], &selectors[0]);
 #endif
 					break;
 				}
@@ -8680,7 +8693,7 @@ namespace basist
 		if (fmt == block_format::cPVRTC1_4_RGB)
 			fixup_pvrtc1_4_modulation_rgb((decoder_etc_block*)pPVRTC_work_mem, pPVRTC_endpoints, pDst_blocks, num_blocks_x, num_blocks_y);
 		else if (fmt == block_format::cPVRTC1_4_RGBA)
-			fixup_pvrtc1_4_modulation_rgba((decoder_etc_block*)pPVRTC_work_mem, pPVRTC_endpoints, pDst_blocks, num_blocks_x, num_blocks_y, pAlpha_blocks, &m_endpoints[0], &m_selectors[0]);
+			fixup_pvrtc1_4_modulation_rgba((decoder_etc_block*)pPVRTC_work_mem, pPVRTC_endpoints, pDst_blocks, num_blocks_x, num_blocks_y, pAlpha_blocks, &endpoints[0], &selectors[0]);
 #endif // BASISD_SUPPORT_PVRTC1
 
 		if (pPVRTC_work_mem)
@@ -10378,46 +10391,84 @@ namespace basist
 
 		if (pHeader->m_tex_format == (int)basis_tex_format::cETC1S)
 		{
-			if (m_lowlevel_etc1s_decoder.m_endpoints.size())
+			if (m_lowlevel_etc1s_decoder.m_local_endpoints.size())
 			{
 				m_lowlevel_etc1s_decoder.clear();
 			}
 
-			if (!pHeader->m_endpoint_cb_file_size || !pHeader->m_selector_cb_file_size || !pHeader->m_tables_file_size)
+			if (pHeader->m_flags & cBASISHeaderFlagUsesGlobalCodebook)
 			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted (0)\n");
+				if (!m_lowlevel_etc1s_decoder.get_global_codebooks())
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: File uses global codebooks, but set_global_codebooks() has not been called\n");
+					return false;
+				}
+				if (!m_lowlevel_etc1s_decoder.get_global_codebooks()->get_endpoints().size())
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: Global codebooks must be unpacked first by calling start_transcoding()\n");
+					return false;
+				}
+				if ((m_lowlevel_etc1s_decoder.get_global_codebooks()->get_endpoints().size() != pHeader->m_total_endpoints) ||
+					 (m_lowlevel_etc1s_decoder.get_global_codebooks()->get_selectors().size() != pHeader->m_total_selectors))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: Global codebook size mismatch (wrong codebooks for file).\n");
+					return false;
+				}
+				if (!pHeader->m_tables_file_size)
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted (2)\n");
+					return false;
+				}
+				if (pHeader->m_tables_file_ofs > data_size)
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (4)\n");
+					return false;
+				}
+				if (pHeader->m_tables_file_size > (data_size - pHeader->m_tables_file_ofs))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (5)\n");
+					return false;
+				}
 			}
-
-			if ((pHeader->m_endpoint_cb_file_ofs > data_size) || (pHeader->m_selector_cb_file_ofs > data_size) || (pHeader->m_tables_file_ofs > data_size))
+			else
 			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (1)\n");
-				return false;
-			}
+				if (!pHeader->m_endpoint_cb_file_size || !pHeader->m_selector_cb_file_size || !pHeader->m_tables_file_size)
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted (0)\n");
+						return false;
+				}
 
-			if (pHeader->m_endpoint_cb_file_size > (data_size - pHeader->m_endpoint_cb_file_ofs))
-			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (2)\n");
-				return false;
-			}
+				if ((pHeader->m_endpoint_cb_file_ofs > data_size) || (pHeader->m_selector_cb_file_ofs > data_size) || (pHeader->m_tables_file_ofs > data_size))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (1)\n");
+					return false;
+				}
 
-			if (pHeader->m_selector_cb_file_size > (data_size - pHeader->m_selector_cb_file_ofs))
-			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (3)\n");
-				return false;
-			}
+				if (pHeader->m_endpoint_cb_file_size > (data_size - pHeader->m_endpoint_cb_file_ofs))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (2)\n");
+					return false;
+				}
 
-			if (pHeader->m_tables_file_size > (data_size - pHeader->m_tables_file_ofs))
-			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (3)\n");
-				return false;
-			}
+				if (pHeader->m_selector_cb_file_size > (data_size - pHeader->m_selector_cb_file_ofs))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (3)\n");
+					return false;
+				}
 
-			if (!m_lowlevel_etc1s_decoder.decode_palettes(
-				pHeader->m_total_endpoints, pDataU8 + pHeader->m_endpoint_cb_file_ofs, pHeader->m_endpoint_cb_file_size,
-				pHeader->m_total_selectors, pDataU8 + pHeader->m_selector_cb_file_ofs, pHeader->m_selector_cb_file_size))
-			{
-				BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: decode_palettes failed\n");
-				return false;
+				if (pHeader->m_tables_file_size > (data_size - pHeader->m_tables_file_ofs))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: file is corrupted or passed in buffer too small (3)\n");
+					return false;
+				}
+
+				if (!m_lowlevel_etc1s_decoder.decode_palettes(
+					pHeader->m_total_endpoints, pDataU8 + pHeader->m_endpoint_cb_file_ofs, pHeader->m_endpoint_cb_file_size,
+					pHeader->m_total_selectors, pDataU8 + pHeader->m_selector_cb_file_ofs, pHeader->m_selector_cb_file_size))
+				{
+					BASISU_DEVEL_ERROR("basisu_transcoder::start_transcoding: decode_palettes failed\n");
+					return false;
+				}
 			}
 
 			if (!m_lowlevel_etc1s_decoder.decode_tables(pDataU8 + pHeader->m_tables_file_ofs, pHeader->m_tables_file_size))
@@ -10429,7 +10480,7 @@ namespace basist
 		else
 		{
 			// Nothing special to do for UASTC.
-			if (m_lowlevel_etc1s_decoder.m_endpoints.size())
+			if (m_lowlevel_etc1s_decoder.m_local_endpoints.size())
 			{
 				m_lowlevel_etc1s_decoder.clear();
 			}

transcoder/basisu_transcoder.h

@@ -169,6 +169,8 @@ namespace basist
 	public:
 		basisu_lowlevel_etc1s_transcoder(const basist::etc1_global_selector_codebook *pGlobal_sel_codebook);
 
+		void set_global_codebooks(const basisu_lowlevel_etc1s_transcoder* pGlobal_codebook) { m_pGlobal_codebook = pGlobal_codebook; }
+		const basisu_lowlevel_etc1s_transcoder *get_global_codebooks() const { return m_pGlobal_codebook; }
 		bool decode_palettes(
 			uint32_t num_endpoints, const uint8_t *pEndpoints_data, uint32_t endpoints_data_size,
 			uint32_t num_selectors, const uint8_t *pSelectors_data, uint32_t selectors_data_size);
@@ -207,8 +209,8 @@ namespace basist
 
 		void clear()
 		{
-			m_endpoints.clear();
-			m_selectors.clear();
+			m_local_endpoints.clear();
+			m_local_selectors.clear();
 			m_endpoint_pred_model.clear();
 			m_delta_endpoint_model.clear();
 			m_selector_model.clear();
@@ -216,12 +218,20 @@ namespace basist
 			m_selector_history_buf_size = 0;
 		}
 
-	private:
+		// Low-level methods
 		typedef basisu::vector<endpoint> endpoint_vec;
-		endpoint_vec m_endpoints;
-
+		const endpoint_vec &get_endpoints() const { return m_local_endpoints; }
+		
 		typedef basisu::vector<selector> selector_vec;
-		selector_vec m_selectors;
+		const selector_vec &get_selectors() const { return m_local_selectors; }
+
+		const etc1_global_selector_codebook* get_global_sel_codebook() const { return m_pGlobal_sel_codebook;	}
+
+	private:
+		const basisu_lowlevel_etc1s_transcoder* m_pGlobal_codebook;
+
+		endpoint_vec m_local_endpoints;
+		selector_vec m_local_selectors;
 
 		const etc1_global_selector_codebook *m_pGlobal_sel_codebook;
 
@@ -487,6 +497,14 @@ namespace basist
 			void* pOutput_blocks, block_format fmt,
 			uint32_t block_stride_in_bytes, uint32_t output_row_pitch_in_blocks_or_pixels);
 
+		void set_global_codebooks(const basisu_lowlevel_etc1s_transcoder* pGlobal_codebook) { m_lowlevel_etc1s_decoder.set_global_codebooks(pGlobal_codebook); }
+		const basisu_lowlevel_etc1s_transcoder* get_global_codebooks() const { return m_lowlevel_etc1s_decoder.get_global_codebooks(); }
+
+		const basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() const { return m_lowlevel_etc1s_decoder; }
+		basisu_lowlevel_etc1s_transcoder& get_lowlevel_etc1s_decoder() { return m_lowlevel_etc1s_decoder; }
+		
+		const basisu_lowlevel_uastc_transcoder& get_lowlevel_uastc_decoder() const { return m_lowlevel_uastc_decoder; }
+		basisu_lowlevel_uastc_transcoder& get_lowlevel_uastc_decoder() { return m_lowlevel_uastc_decoder; }
 	private:
 		mutable basisu_lowlevel_etc1s_transcoder m_lowlevel_etc1s_decoder;
 		mutable basisu_lowlevel_uastc_transcoder m_lowlevel_uastc_decoder;

transcoder/basisu_transcoder_internal.h

@@ -694,6 +694,11 @@ namespace basist
 	{
 		color32 m_color5;
 		uint8_t m_inten5;
+		bool operator== (const endpoint& rhs) const
+		{
+			return (m_color5.r == rhs.m_color5.r) && (m_color5.g == rhs.m_color5.g) && (m_color5.b == rhs.m_color5.b) && (m_inten5 == rhs.m_inten5);
+		}
+		bool operator!= (const endpoint& rhs) const { return !(*this == rhs); }
 	};
 
 	struct selector
@@ -706,6 +711,17 @@ namespace basist
 
 		uint8_t m_lo_selector, m_hi_selector;
 		uint8_t m_num_unique_selectors;
+		bool operator== (const selector& rhs) const
+		{
+			return (m_selectors[0] == rhs.m_selectors[0]) &&
+				(m_selectors[1] == rhs.m_selectors[1]) &&
+				(m_selectors[2] == rhs.m_selectors[2]) &&
+				(m_selectors[3] == rhs.m_selectors[3]);
+		}
+		bool operator!= (const selector& rhs) const
+		{
+			return !(*this == rhs);
+		}
 
 		void init_flags()
 		{