diff --git a/include/bx/commandline.h b/include/bx/commandline.h index 2f75eca..39445b0 100644 --- a/include/bx/commandline.h +++ b/include/bx/commandline.h @@ -10,6 +10,7 @@ namespace bx { + struct AllocatorI; class Error; struct WriterI; @@ -137,6 +138,34 @@ namespace bx int32_t m_numOptions; }; + /// Process arguments as UTF-8. + /// + /// On Windows `main` receives arguments converted to the ANSI code page, which loses every + /// character the code page can't represent, so they are taken from the process command line + /// instead. On other platforms, or when that fails, `_argv` is used as is. + /// + class CommandLineArgs + { + public: + /// + CommandLineArgs(AllocatorI* _allocator, int32_t _argc, const char* const* _argv); + + /// + ~CommandLineArgs(); + + /// + int32_t getArgc() const; + + /// + const char** getArgv() const; + + private: + AllocatorI* m_allocator; + void* m_data; + int32_t m_argc; + const char** m_argv; + }; + } // namespace bx #endif /// BX_COMMANDLINE_H_HEADER_GUARD diff --git a/include/bx/filepath.h b/include/bx/filepath.h index ad40a70..bf62928 100644 --- a/include/bx/filepath.h +++ b/include/bx/filepath.h @@ -41,6 +41,9 @@ namespace bx /// ^^^^^^^ /// +-filename /// + /// Paths are UTF-8 on every platform. On Windows they are converted to UTF-16 before reaching + /// the OS, so the process code page has no effect on them. + /// class FilePath { public: diff --git a/include/bx/os.h b/include/bx/os.h index bba29d8..4c533d0 100644 --- a/include/bx/os.h +++ b/include/bx/os.h @@ -128,7 +128,7 @@ namespace bx template ProtoT dlsym(void* _handle, const StringView& _symbol); - /// Read an environment variable. + /// Read an environment variable. Name and value are UTF-8. /// /// @param[out] _out Buffer that receives the value (may be NULL to query required size). /// @param[in,out] _inOutSize On input the capacity of `_out`, on output the required size including terminator. @@ -137,7 +137,7 @@ namespace bx /// bool getEnv(char* _out, uint32_t* _inOutSize, const StringView& _name); - /// Set an environment variable. + /// Set an environment variable. Name and value are UTF-8. /// /// @param[in] _name Name of the environment variable. /// @param[in] _value Value to assign. @@ -146,7 +146,7 @@ namespace bx /// Change current working directory of the process. /// - /// @param[in] _path Target directory path. + /// @param[in] _path Target directory path, UTF-8. /// @returns 0 on success, non-zero on failure. /// int chdir(const char* _path); diff --git a/include/bx/string.h b/include/bx/string.h index 409c08d..e4facd9 100644 --- a/include/bx/string.h +++ b/include/bx/string.h @@ -549,6 +549,16 @@ namespace bx /// Converts 64-bit unsigned integer value to string. int32_t toString(char* _out, int32_t _max, uint64_t _value, uint32_t _base = 10, char _separator = '\0'); + /// Converts UTF-8 string to NUL terminated UTF-16 string. Returns number of UTF-16 code units + /// written including the terminator, or 0 when the input is not valid UTF-8 or the output doesn't + /// fit. With `_max` 0 returns the size required. + int32_t toUtf16(uint16_t* _out, int32_t _max, const StringView& _str); + + /// Converts NUL terminated UTF-16 string to NUL terminated UTF-8 string. Returns number of bytes + /// written including the terminator, or 0 when the input is not valid UTF-16 or the output + /// doesn't fit. With `_max` 0 returns the size required. + int32_t fromUtf16(char* _out, int32_t _max, const uint16_t* _str); + /// Converts string to bool value. bool fromString(bool* _out, const StringView& _str); diff --git a/src/commandline.cpp b/src/commandline.cpp index 4c615c0..b31052c 100644 --- a/src/commandline.cpp +++ b/src/commandline.cpp @@ -3,11 +3,16 @@ * License: https://github.com/bkaradzic/bx/blob/master/LICENSE */ +#include #include #include #include #include +#if BX_PLATFORM_WINDOWS +extern "C" __declspec(dllimport) wchar_t* __stdcall GetCommandLineW(); +#endif // BX_PLATFORM_WINDOWS + namespace bx { static void skipSpace(Scanner& _scanner, char _term) @@ -662,4 +667,62 @@ namespace bx return NULL; } + CommandLineArgs::CommandLineArgs(AllocatorI* _allocator, int32_t _argc, const char* const* _argv) + : m_allocator(_allocator) + , m_data(NULL) + , m_argc(_argc) + , m_argv(const_cast(_argv) ) + { +#if BX_PLATFORM_WINDOWS + const uint16_t* cmdLine = (const uint16_t*)::GetCommandLineW(); + const int32_t size = fromUtf16(NULL, 0, cmdLine); + + if (0 == size) + { + return; + } + + const int32_t maxArgs = size/2 + 1; + const int32_t bufferSize = size*2 + 2; + + uint8_t* data = (uint8_t*)alloc(m_allocator, maxArgs*sizeof(char*) + size + bufferSize); + + const char** argv = (const char**)data; + char* utf8 = (char*)&data[maxArgs*sizeof(char*)]; + char* buffer = &utf8[size]; + + if (0 == fromUtf16(utf8, size, cmdLine) ) + { + free(m_allocator, data); + return; + } + + uint32_t len = uint32_t(bufferSize); + int32_t argc = 0; + tokenizeCommandLine(StringView(utf8, size-1), buffer, len, argc, (char**)argv, maxArgs); + + m_data = data; + m_argc = argc; + m_argv = argv; +#endif // BX_PLATFORM_WINDOWS + } + + CommandLineArgs::~CommandLineArgs() + { + if (NULL != m_data) + { + free(m_allocator, m_data); + } + } + + int32_t CommandLineArgs::getArgc() const + { + return m_argc; + } + + const char** CommandLineArgs::getArgv() const + { + return m_argv; + } + } // namespace bx diff --git a/src/file.cpp b/src/file.cpp index 3849248..82158c1 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -97,6 +97,24 @@ namespace bx # define ftello64 ftell # endif // BX_ + static FILE* openFile(const FilePath& _filePath, const char* _mode) + { +#if BX_PLATFORM_WINDOWS + wchar_t filePath[kMaxFilePath]; + wchar_t mode[4]; + + if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) + || 0 == toUtf16( (uint16_t*)mode, BX_COUNTOF(mode), _mode) ) + { + return NULL; + } + + return _wfopen(filePath, mode); +#else + return fopen(_filePath.getCPtr(), _mode); +#endif // BX_PLATFORM_WINDOWS + } + class FileReaderImpl : public FileReaderI { public: @@ -121,7 +139,7 @@ namespace bx return false; } - m_file = fopen(_filePath.getCPtr(), "rb"); + m_file = openFile(_filePath, "rb"); if (NULL == m_file) { BX_ERROR_SET(_err, kErrorReaderWriterOpen, "FileReader: Failed to open file."); @@ -201,7 +219,7 @@ namespace bx return false; } - m_file = fopen(_filePath.getCPtr(), _append ? "ab" : "wb"); + m_file = openFile(_filePath, _append ? "ab" : "wb"); if (NULL == m_file) { @@ -579,6 +597,48 @@ namespace bx #if BX_CONFIG_CRT_DIRECTORY_READER +# if BX_PLATFORM_WINDOWS + typedef _WDIR DirHandle; + typedef _wdirent DirEntry; +# else + typedef DIR DirHandle; + typedef dirent DirEntry; +# endif // BX_PLATFORM_WINDOWS + + static DirHandle* openDir(const FilePath& _filePath) + { +#if BX_PLATFORM_WINDOWS + wchar_t filePath[kMaxFilePath]; + + if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) ) + { + return NULL; + } + + return _wopendir(filePath); +#else + return opendir(_filePath.getCPtr() ); +#endif // BX_PLATFORM_WINDOWS + } + + static DirEntry* readDir(DirHandle* _dir) + { +#if BX_PLATFORM_WINDOWS + return _wreaddir(_dir); +#else + return readdir(_dir); +#endif // BX_PLATFORM_WINDOWS + } + + static void closeDir(DirHandle* _dir) + { +#if BX_PLATFORM_WINDOWS + _wclosedir(_dir); +#else + closedir(_dir); +#endif // BX_PLATFORM_WINDOWS + } + class DirectoryReaderImpl : public ReaderOpenI, public CloserI, public ReaderI { public: @@ -597,7 +657,7 @@ namespace bx { BX_ASSERT(NULL != _err, "Reader/Writer interface calling functions must handle errors."); - m_dir = opendir(_filePath.getCPtr() ); + m_dir = openDir(_filePath); if (NULL == m_dir) { @@ -614,7 +674,7 @@ namespace bx { if (NULL != m_dir) { - closedir(m_dir); + closeDir(m_dir); m_dir = NULL; } } @@ -651,11 +711,11 @@ namespace bx return total; } - static bool fetch(FileInfo& _out, DIR* _dir) + static bool fetch(FileInfo& _out, DirHandle* _dir) { for (;;) { - const dirent* item = readdir(_dir); + const DirEntry* item = readDir(_dir); if (NULL == item) { @@ -665,26 +725,40 @@ namespace bx if (0 != (item->d_type & DT_DIR) ) { _out.type = FileType::Dir; - _out.size = UINT64_MAX; - _out.filePath.set(item->d_name); - return true; } - - if (0 != (item->d_type & DT_REG) ) + else if (0 != (item->d_type & DT_REG) ) { _out.type = FileType::File; - _out.size = UINT64_MAX; - _out.filePath.set(item->d_name); - return true; } + else + { + continue; + } + + _out.size = UINT64_MAX; + +#if BX_PLATFORM_WINDOWS + char name[kMaxFilePath]; + + if (0 == fromUtf16(name, BX_COUNTOF(name), (const uint16_t*)item->d_name) ) + { + continue; + } + + _out.filePath.set(name); +#else + _out.filePath.set(item->d_name); +#endif // BX_PLATFORM_WINDOWS + + return true; } return false; } - FileInfo m_cache; - DIR* m_dir; - int32_t m_pos; + FileInfo m_cache; + DirHandle* m_dir; + int32_t m_pos; }; #else @@ -761,9 +835,16 @@ namespace bx _outFileInfo.size = 0; _outFileInfo.type = FileType::Count; -# if BX_COMPILER_MSVC +# if BX_PLATFORM_WINDOWS + wchar_t filePath[kMaxFilePath]; + + if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) ) + { + return false; + } + struct ::_stat64 st; - int32_t result = ::_stat64(_filePath.getCPtr(), &st); + int32_t result = ::_wstat64(filePath, &st); if (0 != result) { @@ -794,7 +875,7 @@ namespace bx { _outFileInfo.type = FileType::Dir; } -# endif // BX_COMPILER_MSVC +# endif // BX_PLATFORM_WINDOWS _outFileInfo.size = st.st_size; @@ -811,10 +892,12 @@ namespace bx return false; } -#if BX_CRT_MSVC - int32_t result = ::_mkdir(_filePath.getCPtr() ); -#elif BX_CRT_MINGW - int32_t result = ::mkdir(_filePath.getCPtr() ); +#if BX_PLATFORM_WINDOWS + wchar_t filePath[kMaxFilePath]; + int32_t result = 0 != toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) + ? ::_wmkdir(filePath) + : -1 + ; #elif BX_CRT_NONE BX_UNUSED(_filePath); int32_t result = -1; @@ -878,23 +961,18 @@ namespace bx return false; } -#if BX_CRT_MSVC || BX_CRT_MINGW +#if BX_PLATFORM_WINDOWS int32_t result = -1; FileInfo fi; - if (stat(fi, _filePath) ) + wchar_t filePath[kMaxFilePath]; + + if (stat(fi, _filePath) + && 0 != toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) ) { - if (FileType::Dir == fi.type) - { -# if BX_CRT_MINGW - result = ::rmdir(_filePath.getCPtr() ); -# else - result = ::_rmdir(_filePath.getCPtr() ); -# endif // BX_CRT_MINGW - } - else - { - result = ::remove(_filePath.getCPtr() ); - } + result = FileType::Dir == fi.type + ? ::_wrmdir(filePath) + : ::_wremove(filePath) + ; } #elif BX_CRT_NONE BX_UNUSED(_filePath); @@ -973,13 +1051,6 @@ namespace bx return remove(_filePath, _err); } -#if BX_PLATFORM_WINDOWS - static int32_t widen(wchar_t* _out, int32_t _max, const FilePath& _filePath) - { - return MultiByteToWideChar(CP_UTF8, 0, _filePath.getCPtr(), -1, _out, _max); - } -#endif // BX_PLATFORM_WINDOWS - bool copy(const FilePath& _from, const FilePath& _to, Error* _err) { BX_ERROR_SCOPE(_err); @@ -1049,8 +1120,8 @@ namespace bx wchar_t from[kMaxFilePath+1]; wchar_t to[kMaxFilePath+1]; - if (0 == widen(from, BX_COUNTOF(from), _from) - || 0 == widen(to, BX_COUNTOF(to), _to) ) + if (0 == toUtf16( (uint16_t*)from, BX_COUNTOF(from), _from) + || 0 == toUtf16( (uint16_t*)to, BX_COUNTOF(to), _to) ) { BX_ERROR_SET(_err, kErrorAccess, "Failed to convert file path."); return false; @@ -1121,7 +1192,7 @@ namespace bx { wchar_t from[kMaxFilePath+1] = {}; - const int32_t num = widen(from, BX_COUNTOF(from)-1, _filePath); + const int32_t num = toUtf16( (uint16_t*)from, BX_COUNTOF(from)-1, _filePath); if (0 == num) { diff --git a/src/filepath.cpp b/src/filepath.cpp index a0cb88d..ec1fce6 100644 --- a/src/filepath.cpp +++ b/src/filepath.cpp @@ -8,17 +8,17 @@ #include #include -#if BX_CRT_MSVC -# include // _getcwd +#if BX_PLATFORM_WINDOWS +# include // _wgetcwd #else # include // getcwd -#endif // BX_CRT_MSVC +#endif // BX_PLATFORM_WINDOWS #if BX_PLATFORM_WINDOWS #if !defined(GetModuleFileName) -extern "C" __declspec(dllimport) unsigned long __stdcall GetModuleFileNameA(void* _module, char* _outFilePath, unsigned long _size); +extern "C" __declspec(dllimport) unsigned long __stdcall GetModuleFileNameW(void* _module, wchar_t* _outFilePath, unsigned long _size); #endif -extern "C" __declspec(dllimport) unsigned long __stdcall GetTempPathA(unsigned long _max, char* _outFilePath); +extern "C" __declspec(dllimport) unsigned long __stdcall GetTempPathW(unsigned long _max, wchar_t* _outFilePath); #elif BX_PLATFORM_OSX extern "C" int _NSGetExecutablePath(char* _buf, uint32_t* _bufSize); #endif // BX_PLATFORM_WINDOWS @@ -167,8 +167,16 @@ namespace bx || BX_CRT_NONE BX_UNUSED(_buffer, _size); return NULL; -#elif BX_CRT_MSVC - return ::_getcwd(_buffer, (int32_t)_size); +#elif BX_PLATFORM_WINDOWS + wchar_t tmp[kMaxFilePath]; + + if (NULL == ::_wgetcwd(tmp, BX_COUNTOF(tmp) ) + || 0 == fromUtf16(_buffer, int32_t(_size), (const uint16_t*)tmp) ) + { + return NULL; + } + + return _buffer; #else return ::getcwd(_buffer, _size); #endif // BX_PLATFORM_* @@ -189,10 +197,18 @@ namespace bx static bool getExecutablePath(char* _out, uint32_t* _inOutSize) { #if BX_PLATFORM_WINDOWS - uint32_t len = ::GetModuleFileNameA(NULL, _out, *_inOutSize); - bool result = len != 0 && len < *_inOutSize; - *_inOutSize = len; - return result; + wchar_t tmp[kMaxFilePath]; + const uint32_t len = ::GetModuleFileNameW(NULL, tmp, BX_COUNTOF(tmp) ); + + if (0 == len + || BX_COUNTOF(tmp) <= len) + { + return false; + } + + const int32_t num = fromUtf16(_out, int32_t(*_inOutSize), (const uint16_t*)tmp); + *_inOutSize = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)tmp)-1; + return 0 != num; #elif BX_PLATFORM_LINUX char tmp[64]; snprintf(tmp, sizeof(tmp), "/proc/%d/exe", getpid() ); @@ -228,10 +244,18 @@ namespace bx static bool getTempPath(char* _out, uint32_t* _inOutSize) { #if BX_PLATFORM_WINDOWS - uint32_t len = ::GetTempPathA(*_inOutSize, _out); - bool result = len != 0 && len < *_inOutSize; - *_inOutSize = len; - return result; + wchar_t tmp[kMaxFilePath]; + const uint32_t len = ::GetTempPathW(BX_COUNTOF(tmp), tmp); + + if (0 == len + || BX_COUNTOF(tmp) <= len) + { + return false; + } + + const int32_t num = fromUtf16(_out, int32_t(*_inOutSize), (const uint16_t*)tmp); + *_inOutSize = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)tmp)-1; + return 0 != num; #else static const StringView s_tmp[] = { diff --git a/src/os.cpp b/src/os.cpp index 5c698ed..9b06ea2 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -6,11 +6,11 @@ #include #include -#if BX_CRT_MSVC -# include -#else +#if BX_PLATFORM_WINDOWS +# include // _wchdir +#elif !BX_CRT_MSVC # include // syscall, _SC_PAGESIZE -#endif // BX_CRT_MSVC +#endif // BX_PLATFORM_WINDOWS #if BX_PLATFORM_WINDOWS || BX_PLATFORM_WINRT # ifndef WIN32_LEAN_AND_MEAN @@ -216,8 +216,15 @@ namespace bx void* dlopen(const FilePath& _filePath) { #if BX_PLATFORM_WINDOWS + wchar_t filePath[kMaxFilePath]; + + if (0 == toUtf16( (uint16_t*)filePath, BX_COUNTOF(filePath), _filePath) ) + { + return NULL; + } + // LOAD_LIBRARY_SEARCH_DEFAULT_DIRS excludes CWD and %PATH% from the search order. - HMODULE handle = ::LoadLibraryExA(_filePath.getCPtr(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + HMODULE handle = ::LoadLibraryExW(filePath, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); BX_WARN(NULL != handle, "dlopen failed: \"%s\" 0x%x.", _filePath.getCPtr(), ::GetLastError() ); @@ -289,8 +296,27 @@ namespace bx strCopy(name, nameMax, _name); #if BX_PLATFORM_WINDOWS - DWORD len = ::GetEnvironmentVariableA(name, _out, *_inOutSize); - bool result = len != 0 && len < *_inOutSize; + wchar_t* wname = (wchar_t*)BX_STACK_ALLOC(nameMax*sizeof(wchar_t) ); + + if (0 == toUtf16( (uint16_t*)wname, nameMax, name) ) + { + *_inOutSize = 0; + return false; + } + + const uint32_t max = NULL != _out ? *_inOutSize : 0; + wchar_t* value = (wchar_t*)BX_STACK_ALLOC(max*sizeof(wchar_t) ); + + DWORD len = ::GetEnvironmentVariableW(wname, value, max); + bool result = len != 0 && len < max; + + if (result) + { + const int32_t num = fromUtf16(_out, int32_t(max), (const uint16_t*)value); + result = 0 != num; + len = 0 != num ? num-1 : fromUtf16(NULL, 0, (const uint16_t*)value); + } + *_inOutSize = len; return result; #elif BX_PLATFORM_EMSCRIPTEN \ @@ -337,7 +363,26 @@ namespace bx } #if BX_PLATFORM_WINDOWS - ::SetEnvironmentVariableA(name, value); + wchar_t* wname = (wchar_t*)BX_STACK_ALLOC(nameMax*sizeof(wchar_t) ); + wchar_t* wvalue = NULL; + + if (0 == toUtf16( (uint16_t*)wname, nameMax, name) ) + { + return; + } + + if (NULL != value) + { + const int32_t valueMax = _value.getLength()+1; + wvalue = (wchar_t*)BX_STACK_ALLOC(valueMax*sizeof(wchar_t) ); + + if (0 == toUtf16( (uint16_t*)wvalue, valueMax, value) ) + { + return; + } + } + + ::SetEnvironmentVariableW(wname, wvalue); #elif BX_PLATFORM_EMSCRIPTEN \ || BX_PLATFORM_PS4 \ || BX_PLATFORM_PS5 \ @@ -367,8 +412,15 @@ namespace bx || BX_CRT_NONE BX_UNUSED(_path); return -1; -#elif BX_CRT_MSVC - return ::_chdir(_path); +#elif BX_PLATFORM_WINDOWS + wchar_t path[kMaxFilePath]; + + if (0 == toUtf16( (uint16_t*)path, BX_COUNTOF(path), _path) ) + { + return -1; + } + + return ::_wchdir(path); #else return ::chdir(_path); #endif // BX_COMPILER_ diff --git a/src/process.cpp b/src/process.cpp index 4208c9f..fc8201e 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -23,10 +23,27 @@ namespace bx #if BX_CONFIG_CRT_PROCESS #if BX_CRT_MSVC -# define popen _popen # define pclose _pclose #endif // BX_CRT_MSVC + static FILE* openProcess(const char* _cmd, const char* _mode) + { +#if BX_PLATFORM_WINDOWS + wchar_t cmd[kMaxFilePath*2]; + wchar_t mode[4]; + + if (0 == toUtf16( (uint16_t*)cmd, BX_COUNTOF(cmd), _cmd) + || 0 == toUtf16( (uint16_t*)mode, BX_COUNTOF(mode), _mode) ) + { + return NULL; + } + + return _wpopen(cmd, mode); +#else + return popen(_cmd, _mode); +#endif // BX_PLATFORM_WINDOWS + } + ProcessReader::ProcessReader() : m_file(NULL) { @@ -52,7 +69,7 @@ namespace bx strCat(tmp, BX_COUNTOF(tmp), "\" "); strCat(tmp, BX_COUNTOF(tmp), _args); - m_file = popen(tmp, "r"); + m_file = openProcess(tmp, "r"); if (NULL == m_file) { BX_ERROR_SET(_err, kErrorReaderWriterOpen, "ProcessReader: Failed to open process."); @@ -123,7 +140,7 @@ namespace bx strCat(tmp, BX_COUNTOF(tmp), "\" "); strCat(tmp, BX_COUNTOF(tmp), _args); - m_file = popen(tmp, "w"); + m_file = openProcess(tmp, "w"); if (NULL == m_file) { BX_ERROR_SET(_err, kErrorReaderWriterOpen, "ProcessWriter: Failed to open process."); diff --git a/src/string.cpp b/src/string.cpp index 2c9d3a9..5a174ef 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -1432,4 +1432,194 @@ namespace bx return formatHumanNumber(_out, _count, double(_value), 0, 1024.0, "iB", "BKMGTPEZY"); } + static uint32_t decodeUtf8(const uint8_t*& _ptr, const uint8_t* _end) + { + const uint8_t ch = *_ptr++; + + if (ch < 0x80) + { + return ch; + } + + uint32_t codePoint; + int32_t num; + + if (0xc0 == (ch & 0xe0) ) + { + codePoint = ch & 0x1f; + num = 1; + } + else if (0xe0 == (ch & 0xf0) ) + { + codePoint = ch & 0x0f; + num = 2; + } + else if (0xf0 == (ch & 0xf8) ) + { + codePoint = ch & 0x07; + num = 3; + } + else + { + return UINT32_MAX; + } + + if (_end - _ptr < num) + { + return UINT32_MAX; + } + + for (int32_t ii = 0; ii < num; ++ii) + { + const uint8_t cc = *_ptr++; + + if (0x80 != (cc & 0xc0) ) + { + return UINT32_MAX; + } + + codePoint = (codePoint << 6) | (cc & 0x3f); + } + + static const uint32_t kMinCodePoint[] = { 0x80, 0x800, 0x10000 }; + + if (codePoint < kMinCodePoint[num-1] + || codePoint > 0x10ffff + || (codePoint >= 0xd800 && codePoint <= 0xdfff) ) + { + return UINT32_MAX; + } + + return codePoint; + } + + int32_t toUtf16(uint16_t* _out, int32_t _max, const StringView& _str) + { + const uint8_t* ptr = (const uint8_t*)_str.getPtr(); + const uint8_t* end = ptr + _str.getLength(); + + int32_t len = 0; + + while (ptr < end) + { + const uint32_t codePoint = decodeUtf8(ptr, end); + + if (UINT32_MAX == codePoint) + { + return 0; + } + + const int32_t num = codePoint < 0x10000 ? 1 : 2; + + if (0 != _max) + { + if (len + num >= _max) + { + return 0; + } + + if (1 == num) + { + _out[len] = uint16_t(codePoint); + } + else + { + const uint32_t vv = codePoint - 0x10000; + _out[len+0] = uint16_t(0xd800 | (vv >> 10) ); + _out[len+1] = uint16_t(0xdc00 | (vv & 0x3ff) ); + } + } + + len += num; + } + + if (0 != _max) + { + _out[len] = 0; + } + + return len+1; + } + + int32_t fromUtf16(char* _out, int32_t _max, const uint16_t* _str) + { + int32_t len = 0; + + for (const uint16_t* ptr = _str; 0 != *ptr;) + { + uint32_t codePoint = *ptr++; + + if (codePoint >= 0xd800 + && codePoint <= 0xdbff) + { + const uint32_t lo = *ptr; + + if (lo < 0xdc00 + || lo > 0xdfff) + { + return 0; + } + + ++ptr; + codePoint = 0x10000 + ( (codePoint - 0xd800) << 10) + (lo - 0xdc00); + } + else if (codePoint >= 0xdc00 + && codePoint <= 0xdfff) + { + return 0; + } + + const int32_t num = + codePoint < 0x80 ? 1 + : codePoint < 0x800 ? 2 + : codePoint < 0x10000 ? 3 + : 4 + ; + + if (0 != _max) + { + if (len + num >= _max) + { + return 0; + } + + char* out = &_out[len]; + + switch (num) + { + case 1: + out[0] = char(codePoint); + break; + + case 2: + out[0] = char(0xc0 | (codePoint >> 6) ); + out[1] = char(0x80 | (codePoint & 0x3f) ); + break; + + case 3: + out[0] = char(0xe0 | (codePoint >> 12) ); + out[1] = char(0x80 | ( (codePoint >> 6) & 0x3f) ); + out[2] = char(0x80 | (codePoint & 0x3f) ); + break; + + default: + out[0] = char(0xf0 | (codePoint >> 18) ); + out[1] = char(0x80 | ( (codePoint >> 12) & 0x3f) ); + out[2] = char(0x80 | ( (codePoint >> 6) & 0x3f) ); + out[3] = char(0x80 | (codePoint & 0x3f) ); + break; + } + } + + len += num; + } + + if (0 != _max) + { + _out[len] = '\0'; + } + + return len+1; + } + } // namespace bx diff --git a/tests/file_test.cpp b/tests/file_test.cpp new file mode 100644 index 0000000..d38c672 --- /dev/null +++ b/tests/file_test.cpp @@ -0,0 +1,127 @@ +/* + * Copyright 2010-2026 Branimir Karadzic. All rights reserved. + * License: https://github.com/bkaradzic/bx/blob/master/LICENSE + */ + +#include "test.h" +#include +#include + +#if BX_PLATFORM_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif // WIN32_LEAN_AND_MEAN +# include +#endif // BX_PLATFORM_WINDOWS + +// Names outside of any single-byte code page, so the ANSI file API cannot represent them. +static const char* kDirName = "bx.test-\xc3\xbc\xe6\xb5\x8b\xe8\xaf\x95"; // u-umlaut, Chinese +static const char* kFileName = "\xd0\xb1\xd1\x80\xd0\xb0\xd0\xbd\xd0\xb8\xd0\xbc\xd0\xb8\xd1\x80.txt"; // Cyrillic +static const char* kMovedName = "\xe6\x96\x87\xe4\xbb\xb6.txt"; // Chinese + +struct ScopedCwd +{ + ScopedCwd() + : m_cwd(bx::Dir::Current) + { + } + + ~ScopedCwd() + { + bx::chdir(m_cwd.getCPtr() ); + } + + bx::FilePath m_cwd; +}; + +TEST_CASE("File paths are UTF-8", "[file][filepath][utf8]") +{ + if (BX_ENABLED(BX_PLATFORM_EMSCRIPTEN) ) + { + SKIP("Not supported by wasm."); + } + + bx::FilePath dir(bx::Dir::Temp); + dir.join(kDirName); + bx::removeAll(dir, bx::ErrorIgnore{}); + + REQUIRE(bx::makeAll(dir, bx::ErrorAssert{}) ); + + bx::FileInfo fi; + REQUIRE(bx::stat(fi, dir) ); + REQUIRE(bx::FileType::Dir == fi.type); + + bx::FilePath file(dir); + file.join(kFileName); + + { + bx::FileWriter writer; + REQUIRE(bx::open(&writer, file, false, bx::ErrorAssert{}) ); + REQUIRE(4 == bx::write(&writer, "utf8", 4, bx::ErrorAssert{}) ); + bx::close(&writer); + } + + REQUIRE(bx::stat(fi, file) ); + REQUIRE(bx::FileType::File == fi.type); + REQUIRE(4 == fi.size); + + { + bx::FileReader reader; + REQUIRE(bx::open(&reader, file, bx::ErrorAssert{}) ); + + char data[4]; + REQUIRE(4 == bx::read(&reader, data, 4, bx::ErrorAssert{}) ); + REQUIRE(0 == bx::memCmp(data, "utf8", 4) ); + bx::close(&reader); + } + + { + bx::DirectoryReader dr; + REQUIRE(bx::open(&dr, dir, bx::ErrorAssert{}) ); + + bool found = false; + bx::Error err; + + while (err.isOk() ) + { + bx::read(&dr, fi, &err); + + if (err.isOk() + && 0 == bx::strCmp(fi.filePath, kFileName) ) + { + found = true; + REQUIRE(bx::FileType::File == fi.type); + } + } + + bx::close(&dr); + REQUIRE(found); + } + + { + ScopedCwd cwd; + REQUIRE(0 == bx::chdir(dir.getCPtr() ) ); + + const bx::FilePath current(bx::Dir::Current); + REQUIRE(0 == bx::strCmp(current.getFileName(), kDirName) ); + + REQUIRE(bx::stat(fi, kFileName) ); + REQUIRE(bx::FileType::File == fi.type); + +#if BX_PLATFORM_WINDOWS + // The name on disk is the Unicode one, not the UTF-8 bytes reinterpreted in the code page. + REQUIRE(INVALID_FILE_ATTRIBUTES != ::GetFileAttributesW(L"\u0431\u0440\u0430\u043d\u0438\u043c\u0438\u0440.txt") ); +#endif // BX_PLATFORM_WINDOWS + } + + bx::FilePath moved(dir); + moved.join(kMovedName); + + REQUIRE(bx::move(file, moved, bx::ErrorAssert{}) ); + REQUIRE(!bx::stat(fi, file) ); + REQUIRE(bx::stat(fi, moved) ); + REQUIRE(4 == fi.size); + + REQUIRE(bx::removeAll(dir, bx::ErrorAssert{}) ); + REQUIRE(!bx::stat(fi, dir) ); +} diff --git a/tests/string_test.cpp b/tests/string_test.cpp index 7fd4c8e..f0c456a 100644 --- a/tests/string_test.cpp +++ b/tests/string_test.cpp @@ -985,3 +985,41 @@ TEST_CASE("StringView is trivially copyable", "[string]") REQUIRE(0 == bx::strCmp(copy.span, "span") ); REQUIRE(0 == bx::strCmp(copy.variant.named.name, "hello") ); } + +TEST_CASE("UTF-16 conversion", "[string][utf8]") +{ + // "a", U+00FC, U+6D4B, U+1F600 (surrogate pair). + static const char* kUtf8 = "a\xc3\xbc\xe6\xb5\x8b\xf0\x9f\x98\x80"; + static const uint16_t kUtf16[] = { 0x0061, 0x00fc, 0x6d4b, 0xd83d, 0xde00, 0 }; + + uint16_t utf16[8]; + REQUIRE(6 == bx::toUtf16(utf16, BX_COUNTOF(utf16), kUtf8) ); + REQUIRE(0 == bx::memCmp(utf16, kUtf16, sizeof(kUtf16) ) ); + + char utf8[16]; + REQUIRE(11 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kUtf16) ); + REQUIRE(0 == bx::strCmp(utf8, kUtf8) ); + + // Required size, and a buffer one short of it. + REQUIRE(6 == bx::toUtf16(NULL, 0, kUtf8) ); + REQUIRE(0 == bx::toUtf16(utf16, 5, kUtf8) ); + REQUIRE(11 == bx::fromUtf16(NULL, 0, kUtf16) ); + REQUIRE(0 == bx::fromUtf16(utf8, 10, kUtf16) ); + + // Empty. + REQUIRE(1 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "") ); + REQUIRE(0 == utf16[0]); + static const uint16_t kEmpty[] = { 0 }; + REQUIRE(1 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kEmpty) ); + REQUIRE('\0' == utf8[0]); + + // Invalid input is rejected rather than replaced. + REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xc3") ); // truncated sequence + REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xc0\x80") ); // overlong NUL + REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xed\xa0\x80") ); // encoded surrogate + REQUIRE(0 == bx::toUtf16(utf16, BX_COUNTOF(utf16), "\xff") ); // not a lead byte + static const uint16_t kLoneHigh[] = { 0xd83d, 0x0061, 0 }; + static const uint16_t kLoneLow[] = { 0xde00, 0 }; + REQUIRE(0 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kLoneHigh) ); + REQUIRE(0 == bx::fromUtf16(utf8, BX_COUNTOF(utf8), kLoneLow) ); +} diff --git a/tests/tokenizecmd_test.cpp b/tests/tokenizecmd_test.cpp index 1386198..c6865cd 100644 --- a/tests/tokenizecmd_test.cpp +++ b/tests/tokenizecmd_test.cpp @@ -4,6 +4,7 @@ */ #include "test.h" +#include #include #include #include @@ -436,3 +437,25 @@ TEST_CASE("tokenizeCommandLine maxArgvs", "") REQUIRE(0 == bx::strCmp(argv[1], "b") ); REQUIRE(0 == bx::strCmp(next, "v g d") ); } + +TEST_CASE("commandLine args", "") +{ + bx::DefaultAllocator allocator; + + const char* argv[] = { "program", "--flag" }; + bx::CommandLineArgs args(&allocator, BX_COUNTOF(argv), argv); + + REQUIRE(0 < args.getArgc() ); + REQUIRE(args.getArgv() != (const char**)NULL); + + if (BX_ENABLED(BX_PLATFORM_WINDOWS) ) + { + // Taken from the process command line, so the program is this test executable. + REQUIRE(!bx::strFind(args.getArgv()[0], "bx.test").isEmpty() ); + } + else + { + REQUIRE(BX_COUNTOF(argv) == args.getArgc() ); + REQUIRE(argv == args.getArgv() ); + } +} diff --git a/tools/bin2c/bin2c.cpp b/tools/bin2c/bin2c.cpp index d1595d7..0b12142 100644 --- a/tools/bin2c/bin2c.cpp +++ b/tools/bin2c/bin2c.cpp @@ -225,7 +225,10 @@ void help(const char* _error = NULL) int main(int _argc, const char* _argv[]) { - bx::CommandLine cmdLine(_argc, _argv, s_options, BX_COUNTOF(s_options) ); + bx::DefaultAllocator allocator; + bx::CommandLineArgs args(&allocator, _argc, _argv); + + bx::CommandLine cmdLine(args.getArgc(), args.getArgv(), s_options, BX_COUNTOF(s_options) ); if (cmdLine.hasArg('h', "help") ) { @@ -278,7 +281,6 @@ int main(int _argc, const char* _argv[]) { size = uint32_t(bx::getSize(&fr) ); - bx::DefaultAllocator allocator; data = bx::alloc(&allocator, size); bx::read(&fr, data, size, bx::ErrorAssert{}); bx::close(&fr);