Scarfski/bx
1
0
Fork 0
mirror of https://github.com/bkaradzic/bx.git synced 2026-10-22 00:31:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Made file paths and command line arguments UTF-8 on Windows. Fixes #438. (#449)

Browse Source
This commit is contained in:
Branimir Karadžić
2026-10-07 19:52:56 -07:00
committed by GitHub
parent f56519b35d
commit 3fd3d3279e
14 changed files with 730 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
include/bx/commandline.h
View File

@@ -10,6 +10,7 @@
namespace bx
{
struct AllocatorI;
class Error;
struct WriterI;
@@ -137,6 +138,34 @@ namespace bx
int32_t m_numOptions;
};
/// Process arguments as UTF-8.
///
/// On Windows `main` receives arguments converted to the ANSI code page, which loses every
/// character the code page can't represent, so they are taken from the process command line
/// instead. On other platforms, or when that fails, `_argv` is used as is.
///
class CommandLineArgs
{
public:
///
CommandLineArgs(AllocatorI* _allocator, int32_t _argc, const char* const* _argv);
///
~CommandLineArgs();
///
int32_t getArgc() const;
///
const char** getArgv() const;
private:
AllocatorI* m_allocator;
void* m_data;
int32_t m_argc;
const char** m_argv;
};
} // namespace bx
#endif /// BX_COMMANDLINE_H_HEADER_GUARD

3
include/bx/filepath.h
View File

@@ -41,6 +41,9 @@ namespace bx
/// ^^^^^^^
/// +-filename
///
/// Paths are UTF-8 on every platform. On Windows they are converted to UTF-16 before reaching
/// the OS, so the process code page has no effect on them.
///
class FilePath
{
public:

6
include/bx/os.h
View File

@@ -128,7 +128,7 @@ namespace bx
template<typename ProtoT>
ProtoT dlsym(void* _handle, const StringView& _symbol);
/// Read an environment variable.
/// Read an environment variable. Name and value are UTF-8.
///
/// @param[out] _out Buffer that receives the value (may be NULL to query required size).
/// @param[in,out] _inOutSize On input the capacity of `_out`, on output the required size including terminator.
@@ -137,7 +137,7 @@ namespace bx
///
bool getEnv(char* _out, uint32_t* _inOutSize, const StringView& _name);
/// Set an environment variable.
/// Set an environment variable. Name and value are UTF-8.
///
/// @param[in] _name Name of the environment variable.
/// @param[in] _value Value to assign.
@@ -146,7 +146,7 @@ namespace bx
/// Change current working directory of the process.
///
/// @param[in] _path Target directory path.
/// @param[in] _path Target directory path, UTF-8.
/// @returns 0 on success, non-zero on failure.
///
int chdir(const char* _path);

10
include/bx/string.h
View File

@@ -549,6 +549,16 @@ namespace bx
/// Converts 64-bit unsigned integer value to string.
int32_t toString(char* _out, int32_t _max, uint64_t _value, uint32_t _base = 10, char _separator = '\0');
/// Converts UTF-8 string to NUL terminated UTF-16 string. Returns number of UTF-16 code units
/// written including the terminator, or 0 when the input is not valid UTF-8 or the output doesn't
/// fit. With `_max` 0 returns the size required.
int32_t toUtf16(uint16_t* _out, int32_t _max, const StringView& _str);
/// Converts NUL terminated UTF-16 string to NUL terminated UTF-8 string. Returns number of bytes
/// written including the terminator, or 0 when the input is not valid UTF-16 or the output
/// doesn't fit. With `_max` 0 returns the size required.
int32_t fromUtf16(char* _out, int32_t _max, const uint16_t* _str);
/// Converts string to bool value.
bool fromString(bool* _out, const StringView& _str);

63
src/commandline.cpp
View File

@@ -3,11 +3,16 @@
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#include <bx/allocator.h>
#include <bx/commandline.h>
#include <bx/readerwriter.h>
#include <bx/scanner.h>
#include <bx/string.h>
#if BX_PLATFORM_WINDOWS
extern "C" __declspec(dllimport) wchar_t* __stdcall GetCommandLineW();
#endif // BX_PLATFORM_WINDOWS
namespace bx
{
static void skipSpace(Scanner& _scanner, char _term)
@@ -662,4 +667,62 @@ namespace bx
return NULL;
}
CommandLineArgs::CommandLineArgs(AllocatorI* _allocator, int32_t _argc, const char* const* _argv)
: m_allocator(_allocator)
, m_data(NULL)
, m_argc(_argc)
, m_argv(const_cast<const char**>(_argv) )
{
#if BX_PLATFORM_WINDOWS
const uint16_t* cmdLine = (const uint16_t*)::GetCommandLineW();
const int32_t size = fromUtf16(NULL, 0, cmdLine);
if (0 == size)
{
return;
}
const int32_t maxArgs = size/2 + 1;
const int32_t bufferSize = size*2 + 2;
uint8_t* data = (uint8_t*)alloc(m_allocator, maxArgs*sizeof(char*) + size + bufferSize);
const char** argv = (const char**)data;
char* utf8 = (char*)&data[maxArgs*sizeof(char*)];
char* buffer = &utf8[size];
if (0 == fromUtf16(utf8, size, cmdLine) )
{
free(m_allocator, data);
return;
}
uint32_t len = uint32_t(bufferSize);
int32_t argc = 0;
tokenizeCommandLine(StringView(utf8, size-1), buffer, len, argc, (char**)argv, maxArgs);
m_data = data;
m_argc = argc;
m_argv = argv;
#endif // BX_PLATFORM_WINDOWS
}
CommandLineArgs::~CommandLineArgs()
{
if (NULL != m_data)
{
free(m_allocator, m_data);
}
}
int32_t CommandLineArgs::getArgc() const
{
return m_argc;
}
const char** CommandLineArgs::getArgv() const
{
return m_argv;
}
} // namespace bx

167
src/file.cpp
View File

@@ -97,6 +97,24 @@ namespace bx
# define ftello64 ftell
# endif // BX_
static FILE* openFile(const FilePath& _filePath, const char* _mode)
{
#if BX_PLATFORM_WINDOWS
wchar_t filePath[kMaxFilePath];
wchar_t mode[4];
if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath)
|| 0 == toUtf16( (uint16_t*)mode, BX_COUNTOF(mode), _mode) )
{
return NULL;
}
return _wfopen(filePath, mode);
#else
return fopen(_filePath.getCPtr(), _mode);
#endif // BX_PLATFORM_WINDOWS
}
class FileReaderImpl : public FileReaderI
{
public:
@@ -121,7 +139,7 @@ namespace bx
return false;
}
m_file = fopen(_filePath.getCPtr(), "rb");
m_file = openFile(_filePath, "rb");
if (NULL == m_file)
{
BX_ERROR_SET(_err, kErrorReaderWriterOpen, "FileReader: Failed to open file.");
@@ -201,7 +219,7 @@ namespace bx
return false;
}
m_file = fopen(_filePath.getCPtr(), _append ? "ab" : "wb");
m_file = openFile(_filePath, _append ? "ab" : "wb");
if (NULL == m_file)
{
@@ -579,6 +597,48 @@ namespace bx
#if BX_CONFIG_CRT_DIRECTORY_READER
# if BX_PLATFORM_WINDOWS
typedef _WDIR DirHandle;
typedef _wdirent DirEntry;
# else
typedef DIR DirHandle;
typedef dirent DirEntry;
# endif // BX_PLATFORM_WINDOWS
static DirHandle* openDir(const FilePath& _filePath)
{
#if BX_PLATFORM_WINDOWS
wchar_t filePath[kMaxFilePath];
if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) )
{
return NULL;
}
return _wopendir(filePath);
#else
return opendir(_filePath.getCPtr() );
#endif // BX_PLATFORM_WINDOWS
}
static DirEntry* readDir(DirHandle* _dir)
{
#if BX_PLATFORM_WINDOWS
return _wreaddir(_dir);
#else
return readdir(_dir);
#endif // BX_PLATFORM_WINDOWS
}
static void closeDir(DirHandle* _dir)
{
#if BX_PLATFORM_WINDOWS
_wclosedir(_dir);
#else
closedir(_dir);
#endif // BX_PLATFORM_WINDOWS
}
class DirectoryReaderImpl : public ReaderOpenI, public CloserI, public ReaderI
{
public:
@@ -597,7 +657,7 @@ namespace bx
{
BX_ASSERT(NULL != _err, "Reader/Writer interface calling functions must handle errors.");
m_dir = opendir(_filePath.getCPtr() );
m_dir = openDir(_filePath);
if (NULL == m_dir)
{
@@ -614,7 +674,7 @@ namespace bx
{
if (NULL != m_dir)
{
closedir(m_dir);
closeDir(m_dir);
m_dir = NULL;
}
}
@@ -651,11 +711,11 @@ namespace bx
return total;
}
static bool fetch(FileInfo& _out, DIR* _dir)
static bool fetch(FileInfo& _out, DirHandle* _dir)
{
for (;;)
{
const dirent* item = readdir(_dir);
const DirEntry* item = readDir(_dir);
if (NULL == item)
{
@@ -665,26 +725,40 @@ namespace bx
if (0 != (item->d_type & DT_DIR) )
{
_out.type = FileType::Dir;
_out.size = UINT64_MAX;
_out.filePath.set(item->d_name);
return true;
}
if (0 != (item->d_type & DT_REG) )
else if (0 != (item->d_type & DT_REG) )
{
_out.type = FileType::File;
_out.size = UINT64_MAX;
_out.filePath.set(item->d_name);
return true;
}
else
{
continue;
}
_out.size = UINT64_MAX;
#if BX_PLATFORM_WINDOWS
char name[kMaxFilePath];
if (0 == fromUtf16(name, BX_COUNTOF(name), (const uint16_t*)item->d_name) )
{
continue;
}
_out.filePath.set(name);
#else
_out.filePath.set(item->d_name);
#endif // BX_PLATFORM_WINDOWS
return true;
}
return false;
}
FileInfo m_cache;
DIR* m_dir;
int32_t m_pos;
FileInfo m_cache;
DirHandle* m_dir;
int32_t m_pos;
};
#else
@@ -761,9 +835,16 @@ namespace bx
_outFileInfo.size = 0;
_outFileInfo.type = FileType::Count;
# if BX_COMPILER_MSVC
# if BX_PLATFORM_WINDOWS
wchar_t filePath[kMaxFilePath];
if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) )
{
return false;
}
struct ::_stat64 st;
int32_t result = ::_stat64(_filePath.getCPtr(), &st);
int32_t result = ::_wstat64(filePath, &st);
if (0 != result)
{
@@ -794,7 +875,7 @@ namespace bx
{
_outFileInfo.type = FileType::Dir;
}
# endif // BX_COMPILER_MSVC
# endif // BX_PLATFORM_WINDOWS
_outFileInfo.size = st.st_size;
@@ -811,10 +892,12 @@ namespace bx
return false;
}
#if BX_CRT_MSVC
int32_t result = ::_mkdir(_filePath.getCPtr() );
#elif BX_CRT_MINGW
int32_t result = ::mkdir(_filePath.getCPtr() );
#if BX_PLATFORM_WINDOWS
wchar_t filePath[kMaxFilePath];
int32_t result = 0 != toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath)
? ::_wmkdir(filePath)
: -1
;
#elif BX_CRT_NONE
BX_UNUSED(_filePath);
int32_t result = -1;
@@ -878,23 +961,18 @@ namespace bx
return false;
}
#if BX_CRT_MSVC || BX_CRT_MINGW
#if BX_PLATFORM_WINDOWS
int32_t result = -1;
FileInfo fi;
if (stat(fi, _filePath) )
wchar_t filePath[kMaxFilePath];
if (stat(fi, _filePath)
&& 0 != toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) )
{
if (FileType::Dir == fi.type)
{
# if BX_CRT_MINGW
result = ::rmdir(_filePath.getCPtr() );
# else
result = ::_rmdir(_filePath.getCPtr() );
# endif // BX_CRT_MINGW
}
else
{
result = ::remove(_filePath.getCPtr() );
}
result = FileType::Dir == fi.type
? ::_wrmdir(filePath)
: ::_wremove(filePath)
;
}
#elif BX_CRT_NONE
BX_UNUSED(_filePath);
@@ -973,13 +1051,6 @@ namespace bx
return remove(_filePath, _err);
}
#if BX_PLATFORM_WINDOWS
static int32_t widen(wchar_t* _out, int32_t _max, const FilePath& _filePath)
{
return MultiByteToWideChar(CP_UTF8, 0, _filePath.getCPtr(), -1, _out, _max);
}
#endif // BX_PLATFORM_WINDOWS
bool copy(const FilePath& _from, const FilePath& _to, Error* _err)
{
BX_ERROR_SCOPE(_err);
@@ -1049,8 +1120,8 @@ namespace bx
wchar_t from[kMaxFilePath+1];
wchar_t to[kMaxFilePath+1];
if (0 == widen(from, BX_COUNTOF(from), _from)
|| 0 == widen(to, BX_COUNTOF(to), _to) )
if (0 == toUtf16( (uint16_t*)from, BX_COUNTOF(from), _from)
|| 0 == toUtf16( (uint16_t*)to, BX_COUNTOF(to), _to) )
{
BX_ERROR_SET(_err, kErrorAccess, "Failed to convert file path.");
return false;
@@ -1121,7 +1192,7 @@ namespace bx
{
wchar_t from[kMaxFilePath+1] = {};
const int32_t num = widen(from, BX_COUNTOF(from)-1, _filePath);
const int32_t num = toUtf16( (uint16_t*)from, BX_COUNTOF(from)-1, _filePath);
if (0 == num)
{

54
src/filepath.cpp
View File

@@ -8,17 +8,17 @@
#include <bx/readerwriter.h>
#include <bx/scanner.h>
#if BX_CRT_MSVC
# include <direct.h> // _getcwd
#if BX_PLATFORM_WINDOWS
# include <direct.h> // _wgetcwd
#else
# include <unistd.h> // getcwd
#endif // BX_CRT_MSVC
#endif // BX_PLATFORM_WINDOWS
#if BX_PLATFORM_WINDOWS
#if !defined(GetModuleFileName)
extern "C" __declspec(dllimport) unsigned long __stdcall GetModuleFileNameA(void* _module, char* _outFilePath, unsigned long _size);
extern "C" __declspec(dllimport) unsigned long __stdcall GetModuleFileNameW(void* _module, wchar_t* _outFilePath, unsigned long _size);
#endif
extern "C" __declspec(dllimport) unsigned long __stdcall GetTempPathA(unsigned long _max, char* _outFilePath);
extern "C" __declspec(dllimport) unsigned long __stdcall GetTempPathW(unsigned long _max, wchar_t* _outFilePath);
#elif BX_PLATFORM_OSX
extern "C" int _NSGetExecutablePath(char* _buf, uint32_t* _bufSize);
#endif // BX_PLATFORM_WINDOWS
@@ -167,8 +167,16 @@ namespace bx
|| BX_CRT_NONE
BX_UNUSED(_buffer, _size);
return NULL;
#elif BX_CRT_MSVC
return ::_getcwd(_buffer, (int32_t)_size);
#elif BX_PLATFORM_WINDOWS
wchar_t tmp[kMaxFilePath];
if (NULL == ::_wgetcwd(tmp, BX_COUNTOF(tmp) )
|| 0 == fromUtf16(_buffer, int32_t(_size), (const uint16_t*)tmp) )
{
return NULL;
}
return _buffer;
#else
return ::getcwd(_buffer, _size);
#endif // BX_PLATFORM_*
@@ -189,10 +197,18 @@ namespace bx
static bool getExecutablePath(char* _out, uint32_t* _inOutSize)
{
#if BX_PLATFORM_WINDOWS
uint32_t len = ::GetModuleFileNameA(NULL, _out, *_inOutSize);
bool result = len != 0 && len < *_inOutSize;
*_inOutSize = len;
return result;
wchar_t tmp[kMaxFilePath];
const uint32_t len = ::GetModuleFileNameW(NULL, tmp, BX_COUNTOF(tmp) );
if (0 == len
|| BX_COUNTOF(tmp) <= len)
{
return false;
}
const int32_t num = fromUtf16(_out, int32_t(*_inOutSize), (const uint16_t*)tmp);
*_inOutSize = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)tmp)-1;
return 0 != num;
#elif BX_PLATFORM_LINUX
char tmp[64];
snprintf(tmp, sizeof(tmp), "/proc/%d/exe", getpid() );
@@ -228,10 +244,18 @@ namespace bx
static bool getTempPath(char* _out, uint32_t* _inOutSize)
{
#if BX_PLATFORM_WINDOWS
uint32_t len = ::GetTempPathA(*_inOutSize, _out);
bool result = len != 0 && len < *_inOutSize;
*_inOutSize = len;
return result;
wchar_t tmp[kMaxFilePath];
const uint32_t len = ::GetTempPathW(BX_COUNTOF(tmp), tmp);
if (0 == len
|| BX_COUNTOF(tmp) <= len)
{
return false;
}
const int32_t num = fromUtf16(_out, int32_t(*_inOutSize), (const uint16_t*)tmp);
*_inOutSize = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)tmp)-1;
return 0 != num;
#else
static const StringView s_tmp[] =
{

72
src/os.cpp
View File

@@ -6,11 +6,11 @@
#include <bx/string.h>
#include <bx/os.h>
#if BX_CRT_MSVC
# include <direct.h>
#else
#if BX_PLATFORM_WINDOWS
# include <direct.h> // _wchdir
#elif !BX_CRT_MSVC
# include <unistd.h> // syscall, _SC_PAGESIZE
#endif // BX_CRT_MSVC
#endif // BX_PLATFORM_WINDOWS
#if BX_PLATFORM_WINDOWS || BX_PLATFORM_WINRT
# ifndef WIN32_LEAN_AND_MEAN
@@ -216,8 +216,15 @@ namespace bx
void* dlopen(const FilePath& _filePath)
{
#if BX_PLATFORM_WINDOWS
wchar_t filePath[kMaxFilePath];
if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) )
{
return NULL;
}
// LOAD_LIBRARY_SEARCH_DEFAULT_DIRS excludes CWD and %PATH% from the search order.
HMODULE handle = ::LoadLibraryExA(_filePath.getCPtr(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
HMODULE handle = ::LoadLibraryExW(filePath, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
BX_WARN(NULL != handle, "dlopen failed: \"%s\" 0x%x.", _filePath.getCPtr(), ::GetLastError() );
@@ -289,8 +296,27 @@ namespace bx
strCopy(name, nameMax, _name);
#if BX_PLATFORM_WINDOWS
DWORD len = ::GetEnvironmentVariableA(name, _out, *_inOutSize);
bool result = len != 0 && len < *_inOutSize;
wchar_t* wname = (wchar_t*)BX_STACK_ALLOC(nameMax*sizeof(wchar_t) );
if (0 == toUtf16( (uint16_t*)wname, nameMax, name) )
{
*_inOutSize = 0;
return false;
}
const uint32_t max = NULL != _out ? *_inOutSize : 0;
wchar_t* value = (wchar_t*)BX_STACK_ALLOC(max*sizeof(wchar_t) );
DWORD len = ::GetEnvironmentVariableW(wname, value, max);
bool result = len != 0 && len < max;
if (result)
{
const int32_t num = fromUtf16(_out, int32_t(max), (const uint16_t*)value);
result = 0 != num;
len = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)value);
}
*_inOutSize = len;
return result;
#elif BX_PLATFORM_EMSCRIPTEN \
@@ -337,7 +363,26 @@ namespace bx
}
#if BX_PLATFORM_WINDOWS
::SetEnvironmentVariableA(name, value);
wchar_t* wname = (wchar_t*)BX_STACK_ALLOC(nameMax*sizeof(wchar_t) );
wchar_t* wvalue = NULL;
if (0 == toUtf16( (uint16_t*)wname, nameMax, name) )
{
return;
}
if (NULL != value)
{
const int32_t valueMax = _value.getLength()+1;
wvalue = (wchar_t*)BX_STACK_ALLOC(valueMax*sizeof(wchar_t) );
if (0 == toUtf16( (uint16_t*)wvalue, valueMax, value) )
{
return;
}
}
::SetEnvironmentVariableW(wname, wvalue);
#elif BX_PLATFORM_EMSCRIPTEN \
|| BX_PLATFORM_PS4 \
|| BX_PLATFORM_PS5 \
@@ -367,8 +412,15 @@ namespace bx
|| BX_CRT_NONE
BX_UNUSED(_path);
return -1;
#elif BX_CRT_MSVC
return ::_chdir(_path);
#elif BX_PLATFORM_WINDOWS
wchar_t path[kMaxFilePath];
if (0 == toUtf16( (uint16_t*)path, BX_COUNTOF(path), _path) )
{
return -1;
}
return ::_wchdir(path);
#else
return ::chdir(_path);
#endif // BX_COMPILER_

23
src/process.cpp
View File

@@ -23,10 +23,27 @@ namespace bx
#if BX_CONFIG_CRT_PROCESS
#if BX_CRT_MSVC
# define popen _popen
# define pclose _pclose
#endif // BX_CRT_MSVC
static FILE* openProcess(const char* _cmd, const char* _mode)
{
#if BX_PLATFORM_WINDOWS
wchar_t cmd[kMaxFilePath*2];
wchar_t mode[4];
if (0 == toUtf16( (uint16_t*)cmd, BX_COUNTOF(cmd), _cmd)
|| 0 == toUtf16( (uint16_t*)mode, BX_COUNTOF(mode), _mode) )
{
return NULL;
}
return _wpopen(cmd, mode);
#else
return popen(_cmd, _mode);
#endif // BX_PLATFORM_WINDOWS
}
ProcessReader::ProcessReader()
: m_file(NULL)
{
@@ -52,7 +69,7 @@ namespace bx
strCat(tmp, BX_COUNTOF(tmp), "\" ");
strCat(tmp, BX_COUNTOF(tmp), _args);
m_file = popen(tmp, "r");
m_file = openProcess(tmp, "r");
if (NULL == m_file)
{
BX_ERROR_SET(_err, kErrorReaderWriterOpen, "ProcessReader: Failed to open process.");
@@ -123,7 +140,7 @@ namespace bx
strCat(tmp, BX_COUNTOF(tmp), "\" ");
strCat(tmp, BX_COUNTOF(tmp), _args);
m_file = popen(tmp, "w");
m_file = openProcess(tmp, "w");
if (NULL == m_file)
{
BX_ERROR_SET(_err, kErrorReaderWriterOpen, "ProcessWriter: Failed to open process.");

190
src/string.cpp
View File

@@ -1432,4 +1432,194 @@ namespace bx
return formatHumanNumber(_out, _count, double(_value), 0, 1024.0, "iB", "BKMGTPEZY");
}
static uint32_t decodeUtf8(const uint8_t*& _ptr, const uint8_t* _end)
{
const uint8_t ch = *_ptr++;
if (ch < 0x80)
{
return ch;
}
uint32_t codePoint;
int32_t num;
if (0xc0 == (ch & 0xe0) )
{
codePoint = ch & 0x1f;
num = 1;
}
else if (0xe0 == (ch & 0xf0) )
{
codePoint = ch & 0x0f;
num = 2;
}
else if (0xf0 == (ch & 0xf8) )
{
codePoint = ch & 0x07;
num = 3;
}
else
{
return UINT32_MAX;
}
if (_end - _ptr < num)
{
return UINT32_MAX;
}
for (int32_t ii = 0; ii < num; ++ii)
{
const uint8_t cc = *_ptr++;
if (0x80 != (cc & 0xc0) )
{
return UINT32_MAX;
}
codePoint = (codePoint << 6) | (cc & 0x3f);
}
static const uint32_t kMinCodePoint[] = { 0x80, 0x800, 0x10000 };
if (codePoint < kMinCodePoint[num-1]
|| codePoint > 0x10ffff
|| (codePoint >= 0xd800 && codePoint <= 0xdfff) )
{
return UINT32_MAX;
}
return codePoint;
}
int32_t toUtf16(uint16_t* _out, int32_t _max, const StringView& _str)
{
const uint8_t* ptr = (const uint8_t*)_str.getPtr();
const uint8_t* end = ptr + _str.getLength();
int32_t len = 0;
while (ptr < end)
{
const uint32_t codePoint = decodeUtf8(ptr, end);
if (UINT32_MAX == codePoint)
{
return 0;
}
const int32_t num = codePoint < 0x10000 ? 1 : 2;
if (0 != _max)
{
if (len + num >= _max)
{
return 0;
}
if (1 == num)
{
_out[len] = uint16_t(codePoint);
}
else
{
const uint32_t vv = codePoint - 0x10000;
_out[len+0] = uint16_t(0xd800 | (vv >> 10) );
_out[len+1] = uint16_t(0xdc00 | (vv & 0x3ff) );
}
}
len += num;
}
if (0 != _max)
{
_out[len] = 0;
}
return len+1;
}
int32_t fromUtf16(char* _out, int32_t _max, const uint16_t* _str)
{
int32_t len = 0;
for (const uint16_t* ptr = _str; 0 != *ptr;)
{
uint32_t codePoint = *ptr++;
if (codePoint >= 0xd800
&& codePoint <= 0xdbff)
{
const uint32_t lo = *ptr;
if (lo < 0xdc00
|| lo > 0xdfff)
{
return 0;
}
++ptr;
codePoint = 0x10000 + ( (codePoint - 0xd800) << 10) + (lo - 0xdc00);
}
else if (codePoint >= 0xdc00
&& codePoint <= 0xdfff)
{
return 0;
}
const int32_t num =
codePoint < 0x80 ? 1
: codePoint < 0x800 ? 2
: codePoint < 0x10000 ? 3
: 4
;
if (0 != _max)
{
if (len + num >= _max)
{
return 0;
}
char* out = &_out[len];
switch (num)
{
case 1:
out[0] = char(codePoint);
break;
case 2:
out[0] = char(0xc0 | (codePoint >> 6) );
out[1] = char(0x80 | (codePoint & 0x3f) );
break;
case 3:
out[0] = char(0xe0 | (codePoint >> 12) );
out[1] = char(0x80 | ( (codePoint >> 6) & 0x3f) );
out[2] = char(0x80 | (codePoint & 0x3f) );
break;
default:
out[0] = char(0xf0 | (codePoint >> 18) );
out[1] = char(0x80 | ( (codePoint >> 12) & 0x3f) );
out[2] = char(0x80 | ( (codePoint >> 6) & 0x3f) );
out[3] = char(0x80 | (codePoint & 0x3f) );
break;
}
}
len += num;
}
if (0 != _max)
{
_out[len] = '\0';
}
return len+1;
}
} // namespace bx

127
tests/file_test.cpp Normal file
View File

@@ -0,0 +1,127 @@
/*
* Copyright 2010-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#include "test.h"
#include <bx/file.h>
#include <bx/os.h>
#if BX_PLATFORM_WINDOWS
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif // WIN32_LEAN_AND_MEAN
# include <windows.h>
#endif // BX_PLATFORM_WINDOWS
// Names outside of any single-byte code page, so the ANSI file API cannot represent them.
static const char* kDirName = "bx.test-\xc3\xbc\xe6\xb5\x8b\xe8\xaf\x95"; // u-umlaut, Chinese
static const char* kFileName = "\xd0\xb1\xd1\x80\xd0\xb0\xd0\xbd\xd0\xb8\xd0\xbc\xd0\xb8\xd1\x80.txt"; // Cyrillic
static const char* kMovedName = "\xe6\x96\x87\xe4\xbb\xb6.txt"; // Chinese
struct ScopedCwd
{
ScopedCwd()
: m_cwd(bx::Dir::Current)
{
}
~ScopedCwd()
{
bx::chdir(m_cwd.getCPtr() );
}
bx::FilePath m_cwd;
};
TEST_CASE("File paths are UTF-8", "[file][filepath][utf8]")
{
if (BX_ENABLED(BX_PLATFORM_EMSCRIPTEN) )
{
SKIP("Not supported by wasm.");
}
bx::FilePath dir(bx::Dir::Temp);
dir.join(kDirName);
bx::removeAll(dir, bx::ErrorIgnore{});
REQUIRE(bx::makeAll(dir, bx::ErrorAssert{}) );
bx::FileInfo fi;
REQUIRE(bx::stat(fi, dir) );
REQUIRE(bx::FileType::Dir == fi.type);
bx::FilePath file(dir);
file.join(kFileName);
{
bx::FileWriter writer;
REQUIRE(bx::open(&writer, file, false, bx::ErrorAssert{}) );
REQUIRE(4 == bx::write(&writer, "utf8", 4, bx::ErrorAssert{}) );
bx::close(&writer);
}
REQUIRE(bx::stat(fi, file) );
REQUIRE(bx::FileType::File == fi.type);
REQUIRE(4 == fi.size);
{
bx::FileReader reader;
REQUIRE(bx::open(&reader, file, bx::ErrorAssert{}) );
char data[4];
REQUIRE(4 == bx::read(&reader, data, 4, bx::ErrorAssert{}) );
REQUIRE(0 == bx::memCmp(data, "utf8", 4) );
bx::close(&reader);
}
{
bx::DirectoryReader dr;
REQUIRE(bx::open(&dr, dir, bx::ErrorAssert{}) );
bool found = false;
bx::Error err;
while (err.isOk() )
{
bx::read(&dr, fi, &err);
if (err.isOk()
&& 0 == bx::strCmp(fi.filePath, kFileName) )
{
found = true;
REQUIRE(bx::FileType::File == fi.type);
}
}
bx::close(&dr);
REQUIRE(found);
}
{
ScopedCwd cwd;
REQUIRE(0 == bx::chdir(dir.getCPtr() ) );
const bx::FilePath current(bx::Dir::Current);
REQUIRE(0 == bx::strCmp(current.getFileName(), kDirName) );
REQUIRE(bx::stat(fi, kFileName) );
REQUIRE(bx::FileType::File == fi.type);
#if BX_PLATFORM_WINDOWS
// The name on disk is the Unicode one, not the UTF-8 bytes reinterpreted in the code page.
REQUIRE(INVALID_FILE_ATTRIBUTES != ::GetFileAttributesW(L"\u0431\u0440\u0430\u043d\u0438\u043c\u0438\u0440.txt") );
#endif // BX_PLATFORM_WINDOWS
}
bx::FilePath moved(dir);
moved.join(kMovedName);
REQUIRE(bx::move(file, moved, bx::ErrorAssert{}) );
REQUIRE(!bx::stat(fi, file) );
REQUIRE(bx::stat(fi, moved) );
REQUIRE(4 == fi.size);
REQUIRE(bx::removeAll(dir, bx::ErrorAssert{}) );
REQUIRE(!bx::stat(fi, dir) );
}

38
tests/string_test.cpp
View File

@@ -985,3 +985,41 @@ TEST_CASE("StringView is trivially copyable", "[string]")
REQUIRE(0 == bx::strCmp(copy.span, "span") );
REQUIRE(0 == bx::strCmp(copy.variant.named.name, "hello") );
}
TEST_CASE("UTF-16 conversion", "[string][utf8]")
{
// "a", U+00FC, U+6D4B, U+1F600 (surrogate pair).
static const char* kUtf8 = "a\xc3\xbc\xe6\xb5\x8b\xf0\x9f\x98\x80";
static const uint16_t kUtf16[] = { 0x0061, 0x00fc, 0x6d4b, 0xd83d, 0xde00, 0 };
uint16_t utf16[8];
REQUIRE(6 == bx::toUtf16(utf16, BX_COUNTOF(utf16), kUtf8) );
REQUIRE(0 == bx::memCmp(utf16, kUtf16, sizeof(kUtf16) ) );
char utf8[16];
REQUIRE(11 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kUtf16) );
REQUIRE(0 == bx::strCmp(utf8, kUtf8) );
// Required size, and a buffer one short of it.
REQUIRE(6 == bx::toUtf16(NULL, 0, kUtf8) );
REQUIRE(0 == bx::toUtf16(utf16, 5, kUtf8) );
REQUIRE(11 == bx::fromUtf16(NULL, 0, kUtf16) );
REQUIRE(0 == bx::fromUtf16(utf8, 10, kUtf16) );
// Empty.
REQUIRE(1 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "") );
REQUIRE(0 == utf16[0]);
static const uint16_t kEmpty[] = { 0 };
REQUIRE(1 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kEmpty) );
REQUIRE('\0' == utf8[0]);
// Invalid input is rejected rather than replaced.
REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xc3") ); // truncated sequence
REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xc0\x80") ); // overlong NUL
REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xed\xa0\x80") ); // encoded surrogate
REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xff") ); // not a lead byte
static const uint16_t kLoneHigh[] = { 0xd83d, 0x0061, 0 };
static const uint16_t kLoneLow[] = { 0xde00, 0 };
REQUIRE(0 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kLoneHigh) );
REQUIRE(0 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kLoneLow) );
}

23
tests/tokenizecmd_test.cpp
View File

@@ -4,6 +4,7 @@
*/
#include "test.h"
#include <bx/allocator.h>
#include <bx/commandline.h>
#include <bx/readerwriter.h>
#include <bx/string.h>
@@ -436,3 +437,25 @@ TEST_CASE("tokenizeCommandLine maxArgvs", "")
REQUIRE(0 == bx::strCmp(argv[1], "b") );
REQUIRE(0 == bx::strCmp(next, "v g d") );
}
TEST_CASE("commandLine args", "")
{
bx::DefaultAllocator allocator;
const char* argv[] = { "program", "--flag" };
bx::CommandLineArgs args(&allocator, BX_COUNTOF(argv), argv);
REQUIRE(0 < args.getArgc() );
REQUIRE(args.getArgv() != (const char**)NULL);
if (BX_ENABLED(BX_PLATFORM_WINDOWS) )
{
// Taken from the process command line, so the program is this test executable.
REQUIRE(!bx::strFind(args.getArgv()[0], "bx.test").isEmpty() );
}
else
{
REQUIRE(BX_COUNTOF(argv) == args.getArgc() );
REQUIRE(argv == args.getArgv() );
}
}

6
tools/bin2c/bin2c.cpp
View File

@@ -225,7 +225,10 @@ void help(const char* _error = NULL)
int main(int _argc, const char* _argv[])
{
bx::CommandLine cmdLine(_argc, _argv, s_options, BX_COUNTOF(s_options) );
bx::DefaultAllocator allocator;
bx::CommandLineArgs args(&allocator, _argc, _argv);
bx::CommandLine cmdLine(args.getArgc(), args.getArgv(), s_options, BX_COUNTOF(s_options) );
if (cmdLine.hasArg('h', "help") )
{
@@ -278,7 +281,6 @@ int main(int _argc, const char* _argv[])
{
size = uint32_t(bx::getSize(&fr) );
bx::DefaultAllocator allocator;
data = bx::alloc(&allocator, size);
bx::read(&fr, data, size, bx::ErrorAssert{});
bx::close(&fr);