From 58d36bd042608b94dde0bb9194f2a2214da0cd54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Sun, 27 Sep 2026 18:32:54 -0700 Subject: [PATCH] Added Class::Word, Class::SpaceHoriz, acceptToken and getTail to Scanner. Added LiteralParser. Added stableSort and insertionSort. (#432) --- include/bx/inline/literal-parser.inl | 27 ++ include/bx/inline/scanner.inl | 25 ++ include/bx/inline/sort.inl | 113 ++++++++ include/bx/literal-parser.h | 139 ++++++++++ include/bx/scanner.h | 28 +- include/bx/sort.h | 47 ++++ include/bx/string.h | 3 + scripts/bx.lua | 1 + src/amalgamated.cpp | 1 + src/literal-parser.cpp | 381 +++++++++++++++++++++++++++ src/scanner.cpp | 12 + src/string.cpp | 13 + tests/literal-parser_test.cpp | 236 +++++++++++++++++ tests/scanner_test.cpp | 109 ++++++++ tests/sort_test.cpp | 196 ++++++++++++++ 15 files changed, 1329 insertions(+), 2 deletions(-) create mode 100644 include/bx/inline/literal-parser.inl create mode 100644 include/bx/literal-parser.h create mode 100644 src/literal-parser.cpp create mode 100644 tests/literal-parser_test.cpp diff --git a/include/bx/inline/literal-parser.inl b/include/bx/inline/literal-parser.inl new file mode 100644 index 0000000..4ddfe68 --- /dev/null +++ b/include/bx/inline/literal-parser.inl @@ -0,0 +1,27 @@ +/* + * Copyright 2011-2026 Branimir Karadzic. All rights reserved. + * License: https://github.com/bkaradzic/bx/blob/master/LICENSE + */ + +#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD +# error "Must be included from bx/literal-parser.h!" +#endif // BX_LITERAL_PARSER_H_HEADER_GUARD + +namespace bx +{ + inline bool LiteralParser::isOk() const + { + return m_ok; + } + + inline const StringView& LiteralParser::getLiteral() const + { + return m_literal; + } + + inline LiteralParser::Type LiteralParser::getType() const + { + return m_type; + } + +} // namespace bx diff --git a/include/bx/inline/scanner.inl b/include/bx/inline/scanner.inl index bbb920b..2849e0d 100644 --- a/include/bx/inline/scanner.inl +++ b/include/bx/inline/scanner.inl @@ -85,6 +85,31 @@ namespace bx return getCursor(); } + inline StringView Scanner::acceptToken(const StringView& _str) + { + if (!hasPrefix(m_tail, _str) ) + { + return getCursor(); + } + + const char last = _str.getTerm()[-1]; + const char* after = m_tail.getPtr() + _str.getLength(); + + if ( (isAlphaNum(last) || '_' == last) + && after < m_input.getTerm() + && (isAlphaNum(*after) || '_' == *after) ) + { + return getCursor(); + } + + return moveTo({ m_tail.getPtr(), _str.getLength() }); + } + + inline StringView Scanner::getTail() const + { + return m_tail; + } + inline StringView Scanner::accept(Class _class) { return moveTo({ m_tail.getPtr(), strFunc(_class).getPtr() }); diff --git a/include/bx/inline/sort.inl b/include/bx/inline/sort.inl index c7f4e61..9996cdc 100644 --- a/include/bx/inline/sort.inl +++ b/include/bx/inline/sort.inl @@ -55,6 +55,119 @@ namespace bx quickSort( (void*)_data, _num, sizeof(Ty), _fn); } + constexpr uint32_t kStableSortInsertionRun = 32; + + template + inline void insertionSort(Ty* _data, uint32_t _num, LessFn _less) + { + for (uint32_t ii = 1; ii < _num; ++ii) + { + Ty item = _data[ii]; + + uint32_t jj = ii; + for (; jj > 0 && _less(item, _data[jj-1]); --jj) + { + _data[jj] = _data[jj-1]; + } + + _data[jj] = item; + } + } + + template + inline void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less) + { + if (2 > _num) + { + return; + } + + for (uint32_t ii = 0; ii < _num; ii += kStableSortInsertionRun) + { + insertionSort(_data + ii, min(kStableSortInsertionRun, _num - ii), _less); + } + + if (kStableSortInsertionRun >= _num) + { + return; + } + + Ty* scratch = (Ty*)alloc(_allocator, _num*sizeof(Ty) ); + for (uint32_t ii = 0; ii < _num; ++ii) + { + BX_PLACEMENT_NEW(&scratch[ii], Ty); + } + + Ty* src = _data; + Ty* dst = scratch; + + for (uint32_t width = kStableSortInsertionRun; width < _num; width <<= 1) + { + for (uint32_t ii = 0; ii < _num; ii += width<<1) + { + const uint32_t mid = min(ii + width, _num); + const uint32_t end = min(ii + (width<<1), _num); + + uint32_t ll = ii; + uint32_t rr = mid; + uint32_t oo = ii; + + while (ll < mid + && rr < end) + { + if (_less(src[rr], src[ll]) ) + { + dst[oo] = src[rr]; + ++rr; + } + else + { + dst[oo] = src[ll]; + ++ll; + } + + ++oo; + } + + for (; ll < mid; ++ll, ++oo) + { + dst[oo] = src[ll]; + } + + for (; rr < end; ++rr, ++oo) + { + dst[oo] = src[rr]; + } + } + + swap(src, dst); + } + + if (src != _data) + { + for (uint32_t ii = 0; ii < _num; ++ii) + { + _data[ii] = src[ii]; + } + } + + for (uint32_t ii = 0; ii < _num; ++ii) + { + scratch[ii].~Ty(); + } + + free(_allocator, scratch); + } + + template + inline void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less) + { + if (!_container.empty() ) + { + stableSort(_allocator, &_container[0], uint32_t(_container.size() ), _less); + } + } + template inline uint32_t unique(void* _data, uint32_t _num, uint32_t _stride, const ComparisonFn _fn) { diff --git a/include/bx/literal-parser.h b/include/bx/literal-parser.h new file mode 100644 index 0000000..7309878 --- /dev/null +++ b/include/bx/literal-parser.h @@ -0,0 +1,139 @@ +/* + * Copyright 2011-2026 Branimir Karadzic. All rights reserved. + * License: https://github.com/bkaradzic/bx/blob/master/LICENSE + */ + +#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD +#define BX_LITERAL_PARSER_H_HEADER_GUARD + +#include + +namespace bx +{ + /// Parses numeric literal at the beginning of a string. + /// + /// Syntax that's accepted is selected by `Profile`, so that numeric literals of different + /// languages can be read by the same parser. Profiles for a few languages are provided by + /// `wgsl`, `glsl`, `hlsl`, `c`, `cpp`, and `msl`. + /// + /// Parsing stops at the first character that can't be part of literal, and trailing input is + /// not an error. Literal that was matched is returned by `getLiteral`, and it's a view into + /// input string. + /// + /// @attention LiteralParser doesn't own input string. Input string must outlive LiteralParser. + /// + class LiteralParser + { + BX_CLASS(LiteralParser + , NO_DEFAULT_CTOR + , NO_COPY + ); + + public: + /// Type of literal that was matched. + /// + enum Type : uint8_t + { + Unknown, //!< Input doesn't start with literal that profile accepts. + + Binary, //!< Binary integer, `0b1010`. + Octal, //!< Octal integer, `0o777`. + Decimal, //!< Decimal integer, `42`. + Hexadecimal, //!< Hexadecimal integer, `0x2a`. + Float, //!< Floating point number, `1.5`, `1e3`, or `0x1.8p3`. + }; + + /// Syntax flags, see `Profile::flags`. + /// + enum : uint32_t + { + Sign = 1u<<0, //!< Leading `+` or `-` is part of literal. + HexPrefix = 1u<<1, //!< Hexadecimal `0x` and `0X` prefix. + BinPrefix = 1u<<2, //!< Binary `0b` prefix. + OctPrefix = 1u<<3, //!< Octal `0o` prefix. + HexFloat = 1u<<4, //!< Hexadecimal float, `0x1.8p3`. It requires `.` or `p`. + LeadingDot = 1u<<5, //!< Float without integer part, `.5`. + TrailingDot = 1u<<6, //!< Float without fractional part, `2.`. + UpperExponent = 1u<<7, //!< `E` and `P` exponent, in addition to `e` and `p`. + PromoteFloatSuffix = 1u<<8, //!< Float suffix makes decimal integer a float, WGSL `1f`. + NoLeadingZero = 1u<<9, //!< Decimal integer may not have leading zero. WGSL rejects `0123`, but `00012.` is a valid float. + }; + + /// Set of suffixes that literal can end with. + /// + struct Suffixes + { + const StringView* data; //!< Suffixes to match. Longest match wins. + uint16_t num; //!< Number of suffixes. + }; + + /// Numeric syntax that LiteralParser accepts. + /// + struct Profile + { + uint32_t flags; //!< Syntax flags. + char separator; //!< Digit separator, or 0 if there is none. For example `_` or `'`. + Suffixes intSuffix; //!< Suffixes accepted on integer literal. + Suffixes floatSuffix; //!< Suffixes accepted on float literal. + }; + + /// Returns profile that accepts sign, binary, octal and hexadecimal prefix, and `_` digit + /// separator, with no suffixes. + /// + static Profile defaults(); + + /// Returns profile for WGSL numeric literals. + /// + static Profile wgsl(); + + /// Returns profile for GLSL numeric literals. + /// + static Profile glsl(); + + /// Returns profile for HLSL numeric literals. + /// + static Profile hlsl(); + + /// Returns profile for C numeric literals. + /// + static Profile c(); + + /// Returns profile for C++ numeric literals. + /// + static Profile cpp(); + + /// Returns profile for Metal Shading Language numeric literals. + /// + static Profile msl(); + + /// Constructor. Parses literal at the beginning of `_str`. + /// + /// @param[in] _str Input string to parse. It's not copied, and it must outlive + /// LiteralParser. + /// @param[in] _profile Numeric syntax to accept. + /// + LiteralParser(const StringView& _str, const Profile& _profile = defaults() ); + + /// Returns true if literal was matched. + /// + bool isOk() const; + + /// Returns literal that was matched, or empty string view if there was no match. + /// + const StringView& getLiteral() const; + + /// Returns type of literal that was matched, or `Type::Unknown` if there was no match. + /// + Type getType() const; + + private: + StringView m_literal; + Type m_type; + bool m_ok; + }; + +} // namespace bx + +#include "inline/literal-parser.inl" + +#endif // BX_LITERAL_PARSER_H_HEADER_GUARD diff --git a/include/bx/scanner.h b/include/bx/scanner.h index 01e2fcf..f5771b0 100644 --- a/include/bx/scanner.h +++ b/include/bx/scanner.h @@ -29,9 +29,11 @@ namespace bx /// enum class Class : uint8_t { - Space, //!< Run of whitespace characters. + Space, //!< Run of whitespace characters, line terminators included. + SpaceHoriz, //!< Run of horizontal whitespace as `isSpaceHoriz` reads it, so up to but not including a line feed. NonSpace, //!< Run of non-whitespace characters. - Identifier, //!< Run of alphanumeric characters and `_`. + Identifier, //!< Identifier as `strIdentifier` reads one: alphanumeric characters and `_`, not starting with a digit. + Word, //!< Word as `strWord` reads one: run of alphanumeric characters and `_`, a leading digit included. EndOfLine, //!< Everything up to, but not including, line terminator. NewLine, //!< Everything up to, and including, line terminator. }; @@ -107,6 +109,19 @@ namespace bx /// StringView accept(const StringView& _str); + /// Accept `_str` if input at cursor starts with it and it does not run into a word. + /// + /// Unlike `accept`, which is a plain prefix test, a token ending in a word + /// character must be followed by something that is not one, so a keyword does not + /// match the head of a longer word: `acceptToken("if")` leaves `ifx` alone. For a + /// token ending in punctuation the two behave the same. + /// + /// @param[in] _str Token to match. + /// + /// @returns Accepted string view, or empty string view if it doesn't match. + /// + StringView acceptToken(const StringView& _str); + /// Accept leading run of characters belonging to `_class`. /// /// @param[in] _class Character class to match. @@ -234,6 +249,15 @@ namespace bx /// StringView getCursor(Cursor _which = Cursor::Current) const; + /// Returns input from cursor to end of input. + /// + /// Where `getCursor` gives a position, this gives the range still to be scanned, + /// which is what another parser over the same text needs handed to it. + /// + /// @returns String view from cursor to end of input. + /// + StringView getTail() const; + /// Returns text between `_from` and current cursor position. /// /// @param[in] _from Position to measure from, usually obtained by `getCursor`. diff --git a/include/bx/sort.h b/include/bx/sort.h index ad1c929..8e3bf0e 100644 --- a/include/bx/sort.h +++ b/include/bx/sort.h @@ -6,6 +6,7 @@ #ifndef BX_SORT_H_HEADER_GUARD #define BX_SORT_H_HEADER_GUARD +#include "allocator.h" #include "bx.h" #include "math.h" #include "string.h" @@ -74,6 +75,52 @@ namespace bx template void quickSort(Ty* _data, uint32_t _num, const ComparisonFn _fn = compareAscending); + /// Performs sort (Insertion Sort algorithm). + /// + /// It's stable, it sorts in place, and it doesn't allocate. Elements move by assignment rather + /// than by byte-wise swap, so unlike `quickSort` element type doesn't have to be trivially + /// move assignable. It's quadratic, and it's meant for short arrays, or as a building block of + /// `stableSort`. + /// + /// @param _data Pointer to array data. + /// @param _num Number of elements. + /// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a + /// strict weak ordering. + /// + template + void insertionSort(Ty* _data, uint32_t _num, LessFn _less); + + /// Performs stable sort: elements that compare equal keep their relative order. + /// + /// Insertion sort for short runs, and bottom-up merge for the rest, ping-ponging between input + /// and a single scratch buffer. Elements move by assignment rather than by byte-wise swap, so + /// unlike `quickSort` element type doesn't have to be trivially move assignable. An element + /// holding a small-string buffer, or anything else self-referential, is safe here and is not + /// safe there. + /// + /// @param _allocator Allocator for scratch buffer. Nothing is allocated when there are few + /// enough elements for insertion sort alone. + /// @param _data Pointer to array data. + /// @param _num Number of elements. + /// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a + /// strict weak ordering. + /// + /// @attention Element type must be default constructible and copy assignable. + /// + template + void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less); + + /// Performs stable sort over contiguous container. + /// + /// @param _allocator Allocator for scratch buffer. + /// @param _container Container with `empty`, `size` and `operator[]`, such as + /// `tinystl::vector`. + /// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a + /// strict weak ordering. + /// + template + void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less); + /// Performs reordering of duplicate elements in the array in the way that unique elements /// are sorted to the front of array, and duplicates are after the return value index. /// diff --git a/include/bx/string.h b/include/bx/string.h index 4575906..e5efc22 100644 --- a/include/bx/string.h +++ b/include/bx/string.h @@ -420,6 +420,9 @@ namespace bx /// Returns string view with whitespace characters trimmed from left. StringView strLTrimSpace(const StringView& _str); + /// Returns string view with horizontal whitespace characters trimmed from left. + StringView strLTrimSpaceHoriz(const StringView& _str); + /// Returns string view with non-whitespace characters trimmed from left. StringView strLTrimNonSpace(const StringView& _str); diff --git a/scripts/bx.lua b/scripts/bx.lua index e7004dc..f84d202 100644 --- a/scripts/bx.lua +++ b/scripts/bx.lua @@ -78,6 +78,7 @@ project "bx" path.join(BX_DIR, "src/file.cpp"), path.join(BX_DIR, "src/filepath.cpp"), path.join(BX_DIR, "src/hash.cpp"), + path.join(BX_DIR, "src/literal-parser.cpp"), path.join(BX_DIR, "src/math.cpp"), path.join(BX_DIR, "src/mutex.cpp"), path.join(BX_DIR, "src/os.cpp"), diff --git a/src/amalgamated.cpp b/src/amalgamated.cpp index 1d679f8..1210681 100644 --- a/src/amalgamated.cpp +++ b/src/amalgamated.cpp @@ -14,6 +14,7 @@ #include "file.cpp" #include "filepath.cpp" #include "hash.cpp" +#include "literal-parser.cpp" #include "math.cpp" #include "mutex.cpp" #include "os.cpp" diff --git a/src/literal-parser.cpp b/src/literal-parser.cpp new file mode 100644 index 0000000..ca339ba --- /dev/null +++ b/src/literal-parser.cpp @@ -0,0 +1,381 @@ +/* + * Copyright 2011-2026 Branimir Karadzic. All rights reserved. + * License: https://github.com/bkaradzic/bx/blob/master/LICENSE + */ + +#include +#include + +namespace bx +{ + namespace + { + static constexpr StringView kWgslInt[] = { "i", "u", "li", "lu" }; + static constexpr StringView kWgslFloat[] = { "f", "h", "lf" }; + static constexpr StringView kGlslInt[] = { "u", "U", "lu", "LU" }; + static constexpr StringView kGlslFloat[] = { "f", "F", "lf", "LF" }; + static constexpr StringView kHlslInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "UL", "ull", "llu", "ULL", "LLU" }; + static constexpr StringView kHlslFloat[] = { "f", "F", "h", "H" }; + static constexpr StringView kCInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "ull", "llu", "ULL", "LLU" }; + static constexpr StringView kCFloat[] = { "f", "F", "l", "L" }; + static constexpr StringView kMslFloat[] = { "f", "F", "h", "H" }; + + using LiteralProfile = LiteralParser::Profile; + using LiteralSuffixes = LiteralParser::Suffixes; + using LiteralType = LiteralParser::Type; + + char peekChar(Scanner& _scanner) + { + const StringView view = _scanner.peek(); + + return view.isEmpty() ? '\0' : *view.getPtr(); + } + + bool acceptDigits(Scanner& _scanner, CharTestFn _fn, char _separator) + { + bool ok = false; + for (char ch = peekChar(_scanner) + ; _fn(ch) || (0 != _separator && _separator == ch) + ; ch = peekChar(_scanner) ) + { + ok |= _fn(ch); + _scanner.accept(); + } + + return ok; + } + + bool acceptSuffix(Scanner& _scanner, const LiteralSuffixes& _suffixes) + { + int32_t best = -1; + int32_t bestLen = 0; + for (uint16_t ii = 0; ii < _suffixes.num; ++ii) + { + const int32_t len = _suffixes.data[ii].getLength(); + if (len > bestLen + && !_scanner.peek(_suffixes.data[ii]).isEmpty() ) + { + best = ii; + bestLen = len; + } + } + + if (0 <= best) + { + _scanner.accept(_suffixes.data[best]); + + return true; + } + + return false; + } + + enum ExponentResult + { + ExponentNone, + ExponentOk, + ExponentBad, + }; + + ExponentResult acceptExponent(Scanner& _scanner, const LiteralProfile& _profile, CharTestFn _digits) + { + const char ch = peekChar(_scanner); + const bool isExponent = 'e' == ch + || (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch) + ; + + if (!isExponent) + { + return ExponentNone; + } + + _scanner.accept(); + _scanner.accept('-', '+'); + + return acceptDigits(_scanner, _digits, _profile.separator) ? ExponentOk : ExponentBad; + } + + LiteralType parseDecimalFloat(Scanner& _scanner, const LiteralProfile& _profile) + { + bool dot = false; + bool frac = false; + if (!_scanner.accept('.').isEmpty() ) + { + dot = true; + frac = acceptDigits(_scanner, isNumeric, _profile.separator); + } + + const ExponentResult exponent = acceptExponent(_scanner, _profile, isNumeric); + if (ExponentBad == exponent) + { + return LiteralType::Unknown; + } + + const bool ok = frac + || ExponentOk == exponent + || (dot && 0 != (_profile.flags & LiteralParser::TrailingDot) ) + ; + + if (!ok) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.floatSuffix); + + return LiteralType::Float; + } + + LiteralType parseAfterInteger(Scanner& _scanner, const LiteralProfile& _profile) + { + const char ch = peekChar(_scanner); + const bool isExponent = 'e' == ch + || (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch) + ; + + if ('.' == ch + || isExponent) + { + return parseDecimalFloat(_scanner, _profile); + } + + if (0 != (_profile.flags & LiteralParser::PromoteFloatSuffix) + && acceptSuffix(_scanner, _profile.floatSuffix) ) + { + return LiteralType::Float; + } + + acceptSuffix(_scanner, _profile.intSuffix); + + return LiteralType::Decimal; + } + + LiteralType parseHex(Scanner& _scanner, const LiteralProfile& _profile) + { + const bool anyHex = acceptDigits(_scanner, isHexNum, _profile.separator); + + bool isFloat = false; + bool anyHexFrac = false; + if (0 != (_profile.flags & LiteralParser::HexFloat) ) + { + if (!_scanner.accept('.').isEmpty() ) + { + isFloat = true; + anyHexFrac = acceptDigits(_scanner, isHexNum, _profile.separator); + } + + const char ch = peekChar(_scanner); + if ('p' == ch + || 'P' == ch) + { + isFloat = true; + _scanner.accept(); + _scanner.accept('-', '+'); + + if (!acceptDigits(_scanner, isNumeric, _profile.separator) ) + { + return LiteralType::Unknown; + } + } + } + + if (isFloat) + { + // `0x.p2` has no significand digits. + if (!anyHex + && !anyHexFrac) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.floatSuffix); + + return LiteralType::Float; + } + + // `0x` with no digits. + if (!anyHex) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.intSuffix); + + return LiteralType::Hexadecimal; + } + + LiteralType parseLiteral(Scanner& _scanner, const LiteralProfile& _profile) + { + if (0 != (_profile.flags & LiteralParser::Sign) ) + { + _scanner.accept('-', '+'); + } + + if (!_scanner.accept('0').isEmpty() ) + { + if (0 != (_profile.flags & LiteralParser::BinPrefix) + && !_scanner.accept('b').isEmpty() ) + { + if (!acceptDigits(_scanner, isBinNum, _profile.separator) ) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.intSuffix); + + return LiteralType::Binary; + } + + if (0 != (_profile.flags & LiteralParser::OctPrefix) + && !_scanner.accept('o').isEmpty() ) + { + if (!acceptDigits(_scanner, isOctNum, _profile.separator) ) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.intSuffix); + + return LiteralType::Octal; + } + + if (0 != (_profile.flags & LiteralParser::HexPrefix) + && !_scanner.accept('x', 'X').isEmpty() ) + { + return parseHex(_scanner, _profile); + } + + const bool leadingZeros = acceptDigits(_scanner, isNumeric, _profile.separator); + const LiteralType type = parseAfterInteger(_scanner, _profile); + + if (leadingZeros + && LiteralType::Decimal == type + && 0 != (_profile.flags & LiteralParser::NoLeadingZero) ) + { + return LiteralType::Unknown; + } + + return type; + } + + if (isNumeric(peekChar(_scanner) ) ) + { + acceptDigits(_scanner, isNumeric, _profile.separator); + + return parseAfterInteger(_scanner, _profile); + } + + if (0 != (_profile.flags & LiteralParser::LeadingDot) + && '.' == peekChar(_scanner) ) + { + _scanner.accept('.'); + + // `.` must be followed by a digit. + if (!acceptDigits(_scanner, isNumeric, _profile.separator) ) + { + return LiteralType::Unknown; + } + + if (ExponentBad == acceptExponent(_scanner, _profile, isNumeric) ) + { + return LiteralType::Unknown; + } + + acceptSuffix(_scanner, _profile.floatSuffix); + + return LiteralType::Float; + } + + return LiteralType::Unknown; + } + + } // namespace + + LiteralParser::Profile LiteralParser::defaults() + { + return + { + Sign | BinPrefix | OctPrefix | HexPrefix, + '_', + { NULL, 0 }, + { NULL, 0 }, + }; + } + + LiteralParser::Profile LiteralParser::wgsl() + { + return + { + HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix | NoLeadingZero, + 0, + { kWgslInt, uint16_t(BX_COUNTOF(kWgslInt) ) }, + { kWgslFloat, uint16_t(BX_COUNTOF(kWgslFloat) ) }, + }; + } + + LiteralParser::Profile LiteralParser::glsl() + { + return + { + HexPrefix | LeadingDot | TrailingDot | UpperExponent, + 0, + { kGlslInt, uint16_t(BX_COUNTOF(kGlslInt) ) }, + { kGlslFloat, uint16_t(BX_COUNTOF(kGlslFloat) ) }, + }; + } + + LiteralParser::Profile LiteralParser::hlsl() + { + return + { + HexPrefix | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix, + 0, + { kHlslInt, uint16_t(BX_COUNTOF(kHlslInt) ) }, + { kHlslFloat, uint16_t(BX_COUNTOF(kHlslFloat) ) }, + }; + } + + LiteralParser::Profile LiteralParser::c() + { + return + { + HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent, + 0, + { kCInt, uint16_t(BX_COUNTOF(kCInt) ) }, + { kCFloat, uint16_t(BX_COUNTOF(kCFloat) ) }, + }; + } + + LiteralParser::Profile LiteralParser::cpp() + { + Profile profile = c(); + profile.flags |= BinPrefix; + profile.separator = '\''; + + return profile; + } + + LiteralParser::Profile LiteralParser::msl() + { + Profile profile = cpp(); + profile.floatSuffix = { kMslFloat, uint16_t(BX_COUNTOF(kMslFloat) ) }; + + return profile; + } + + LiteralParser::LiteralParser(const StringView& _str, const Profile& _profile) + : m_type(Type::Unknown) + , m_ok(false) + { + Scanner scanner(_str); + const StringView start = scanner.getCursor(); + + m_type = parseLiteral(scanner, _profile); + m_ok = Type::Unknown != m_type; + m_literal = m_ok + ? scanner.between(start) + : StringView() + ; + } + +} // namespace bx diff --git a/src/scanner.cpp b/src/scanner.cpp index e1f3069..34ae66e 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -90,6 +90,9 @@ namespace bx case Class::NonSpace: return strLTrimNonSpace(m_tail); + case Class::SpaceHoriz: + return strLTrimSpaceHoriz(m_tail); + case Class::Identifier: { const StringView word = strIdentifier(m_tail); @@ -99,6 +102,15 @@ namespace bx ; } + case Class::Word: + { + const StringView word = strWord(m_tail); + return word.isEmpty() + ? m_tail + : StringView(word.getTerm(), m_tail.getTerm() ) + ; + } + case Class::EndOfLine: return strFindEol(m_tail); diff --git a/src/string.cpp b/src/string.cpp index a4a290c..2c9d3a9 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -410,6 +410,19 @@ namespace bx return StringView(_str.getTerm(), _str.getTerm() ); } + StringView strLTrimSpaceHoriz(const StringView& _str) + { + for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr) + { + if (!isSpaceHoriz(*ptr) ) + { + return StringView(ptr, term); + } + } + + return StringView(_str.getTerm(), _str.getTerm() ); + } + StringView strLTrimNonSpace(const StringView& _str) { for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr) diff --git a/tests/literal-parser_test.cpp b/tests/literal-parser_test.cpp new file mode 100644 index 0000000..52817e3 --- /dev/null +++ b/tests/literal-parser_test.cpp @@ -0,0 +1,236 @@ +/* + * Copyright 2011-2026 Branimir Karadzic. All rights reserved. + * License: https://github.com/bkaradzic/bx/blob/master/LICENSE + */ + +#include "test.h" + +#include + +bool isNumber(bx::StringView _str) +{ + const char* str = _str.getPtr(); + int32_t len = _str.getLength(); + + const char* end = str + len; + const char* p = str; + + if (p < end && (*p == '-' || *p == '+')) + { + p++; + } + + bool has_digit = false; + + while (p < end && isdigit((unsigned char)*p)) + { + has_digit = true; + p++; + } + + if (!has_digit) return false; + + if (p < end && *p == '.') + { + p++; + has_digit = false; + + while (p < end && isdigit((unsigned char)*p)) + { + has_digit = true; + p++; + } + + if (!has_digit) return false; + } + + if (p < end && (*p == 'e' || *p == 'E')) + { + p++; + + if (p < end && (*p == '-' || *p == '+')) + { + p++; + } + + has_digit = false; + + while (p < end && isdigit((unsigned char)*p)) + { + has_digit = true; + p++; + } + + if (!has_digit) return false; + } + + return p == end; +} + +bool testNumber(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown) +{ + bx::LiteralParser p(_input); + + const bx::StringView literal = p.getLiteral(); + + bool ok = true + && p.isOk() + && _type == p.getType() + && bx::isEqual(literal, _output) + ; + + if (!ok + && _type != bx::LiteralParser::Type::Unknown) + { + DBG("%d: %S, expected %d: %S", p.getType(), &literal, _type, &_output); + } + + BX_TRACE("is_number %d, %d <- %S" + , isNumber(literal) + , ok + , &literal + ); + + return ok; +} + +bool testNumber(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown) +{ + return testNumber(_inputAndOutput, _inputAndOutput, _type); +} + +TEST_CASE("LiteralParser", "[parser]") +{ + // error + REQUIRE(!testNumber("") ); + REQUIRE(!testNumber(".") ); + REQUIRE(!testNumber("abvgd") ); + REQUIRE(!testNumber("-e0") ); + REQUIRE(!testNumber("-e-89") ); + REQUIRE(!testNumber("0b") ); + REQUIRE(!testNumber("0o") ); + REQUIRE(!testNumber("0x") ); + REQUIRE(!testNumber("0e") ); + REQUIRE(!testNumber("-0.e-") ); + REQUIRE(!testNumber("1.38913e-") ); + + // decimal + REQUIRE(testNumber("0", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("00", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("000", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0000", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("00_00", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0_0_0_0", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0-", "0", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0-e0", "0", bx::LiteralParser::Type::Decimal) ); + + // binary + REQUIRE(testNumber("0b0001001110001001", bx::LiteralParser::Type::Binary) ); + REQUIRE(testNumber("0b0001_0011_1000_1001", bx::LiteralParser::Type::Binary) ); + REQUIRE(testNumber("0b0;", "0b0", bx::LiteralParser::Type::Binary) ); + REQUIRE(testNumber("0b0+", "0b0", bx::LiteralParser::Type::Binary) ); + + // octal + REQUIRE(testNumber("02555", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0o2555", bx::LiteralParser::Type::Octal) ); + REQUIRE(testNumber("0o25_55", bx::LiteralParser::Type::Octal) ); + + // hex + REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0x0;", "0x0", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0x0+", "0x0", bx::LiteralParser::Type::Hexadecimal) ); + + // integer + REQUIRE(testNumber("1389", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("13_89", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) ); + REQUIRE(testNumber("0+", "0", bx::LiteralParser::Type::Decimal) ); + + // floating point + REQUIRE(testNumber("0.1389", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("-0.1389", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("+0.1389", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("0.e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("-0.e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("+0.e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("138e-9", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("1.38913e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("1.389e+001", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("1.389e-002", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("1.389e-002;", "1.389e-002", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("-0.e-89;", "-0.e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("-0.e-89+", "-0.e-89", bx::LiteralParser::Type::Float) ); + REQUIRE(testNumber("-0.e-89e", "-0.e-89", bx::LiteralParser::Type::Float) ); + + // hex with letters (lower/upper/mixed) and digit separators + REQUIRE(testNumber("0xabcdef", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xABCDEF", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xDeadC0de", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xff", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0x13_89", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xFF_FF", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xdead;", "0xdead", bx::LiteralParser::Type::Hexadecimal) ); + REQUIRE(testNumber("0xC0DE-", "0xC0DE", bx::LiteralParser::Type::Hexadecimal) ); + + // more errors: a lone sign or a leading separator is not a number + REQUIRE(!testNumber("+") ); + REQUIRE(!testNumber("-") ); + REQUIRE(!testNumber("_5") ); + REQUIRE(!testNumber("0xg") ); +} + +static bool testWgsl(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type) +{ + bx::LiteralParser p(_input, bx::LiteralParser::wgsl() ); + + return p.isOk() + && _type == p.getType() + && bx::isEqual(p.getLiteral(), _output) + ; +} + +static bool testWgsl(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type) +{ + return testWgsl(_inputAndOutput, _inputAndOutput, _type); +} + +TEST_CASE("LiteralParser WGSL profile", "[parser]") +{ + typedef bx::LiteralParser LP; + + // integers (+ WGSL i/u/li/lu suffixes) and hex + REQUIRE(testWgsl("0", LP::Decimal) ); + REQUIRE(testWgsl("42", LP::Decimal) ); + REQUIRE(testWgsl("1u", LP::Decimal) ); + REQUIRE(testWgsl("1i", LP::Decimal) ); + REQUIRE(testWgsl("1li", LP::Decimal) ); + REQUIRE(testWgsl("1lu", LP::Decimal) ); + REQUIRE(testWgsl("0xFF", LP::Hexadecimal) ); + REQUIRE(testWgsl("0xffu", LP::Hexadecimal) ); + + // floats: fraction/exponent, edge dots, suffixes, hex float + REQUIRE(testWgsl("1.0", LP::Float) ); + REQUIRE(testWgsl(".5", LP::Float) ); + REQUIRE(testWgsl("2.", LP::Float) ); + REQUIRE(testWgsl("1e5", LP::Float) ); + REQUIRE(testWgsl("3.14e-2", LP::Float) ); + REQUIRE(testWgsl("1.5f", LP::Float) ); + REQUIRE(testWgsl("1f", LP::Float) ); // bare integer + float suffix -> float + REQUIRE(testWgsl("1h", LP::Float) ); + REQUIRE(testWgsl("1lf", LP::Float) ); + REQUIRE(testWgsl("0x1p-3", LP::Float) ); + REQUIRE(testWgsl("0x1.8p3", LP::Float) ); + + // prefix match: trailing input is not part of the literal + REQUIRE(testWgsl("42;", "42", LP::Decimal) ); + REQUIRE(testWgsl("0xFFu-", "0xFFu", LP::Hexadecimal) ); + REQUIRE(testWgsl("1.x", "1.", LP::Float) ); // "1." is a float (trailing dot), then ".x" + + // WGSL has no sign, no 0b/0o prefixes, no '_' separators + REQUIRE(!bx::LiteralParser("-5", LP::wgsl() ).isOk() ); // sign is a separate token + REQUIRE(testWgsl("0b1", "0", LP::Decimal) ); // "0" then "b1" + REQUIRE(testWgsl("5_5", "5", LP::Decimal) ); // "5" then "_5" +} diff --git a/tests/scanner_test.cpp b/tests/scanner_test.cpp index db79149..0dd9787 100644 --- a/tests/scanner_test.cpp +++ b/tests/scanner_test.cpp @@ -390,3 +390,112 @@ TEST_CASE("LineReader", "[scanner][string]") REQUIRE(second.getPtr() == input.getPtr() + 4); } } + +TEST_CASE("Scanner.class Word", "[scanner]") +{ + { + bx::Scanner sc("2d, f32"); + + REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) ); + REQUIRE("," == sc.accept(',') ); + } + + { + bx::Scanner sc("2d"); + + REQUIRE(sc.accept(bx::Scanner::Class::Identifier).isEmpty() ); + REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) ); + } + + { + bx::Scanner sc("_leading and9digits"); + + REQUIRE("_leading" == sc.accept(bx::Scanner::Class::Word) ); + REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) ); + REQUIRE("and9digits" == sc.accept(bx::Scanner::Class::Word) ); + REQUIRE(sc.isDone() ); + } +} + +TEST_CASE("Scanner.class SpaceHoriz", "[scanner]") +{ + { + bx::Scanner horiz(" \t\nx"); + + REQUIRE(" \t" == horiz.accept(bx::Scanner::Class::SpaceHoriz) ); + REQUIRE("\n" == horiz.accept('\n') ); + + bx::Scanner space(" \t\nx"); + + REQUIRE(" \t\n" == space.accept(bx::Scanner::Class::Space) ); + } + + { + bx::Scanner sc("x"); + + REQUIRE(sc.accept(bx::Scanner::Class::SpaceHoriz).isEmpty() ); + REQUIRE("x" == sc.accept('x') ); + } + + { + bx::Scanner sc("a \r\nb"); + + REQUIRE("a" == sc.accept(bx::Scanner::Class::Word) ); + REQUIRE(" \r" == sc.accept(bx::Scanner::Class::SpaceHoriz) ); + REQUIRE("\n" == sc.accept('\n') ); + REQUIRE("b" == sc.accept(bx::Scanner::Class::Word) ); + } +} + +TEST_CASE("Scanner.acceptToken", "[scanner]") +{ + { + bx::Scanner bare("ifx"); + REQUIRE("if" == bare.accept(bx::StringView("if") ) ); + + bx::Scanner token("ifx"); + REQUIRE(token.acceptToken(bx::StringView("if") ).isEmpty() ); + REQUIRE("ifx" == token.accept(bx::Scanner::Class::Word) ); + } + + { + bx::Scanner sc("if (x)"); + + REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) ); + REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) ); + REQUIRE("(" == sc.acceptToken(bx::StringView("(") ) ); + } + + { + bx::Scanner sc("->x"); + + REQUIRE("->" == sc.acceptToken(bx::StringView("->") ) ); + REQUIRE("x" == sc.accept(bx::Scanner::Class::Word) ); + } + + { + bx::Scanner sc("if"); + + REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) ); + REQUIRE(sc.isDone() ); + } +} + +TEST_CASE("Scanner.getTail", "[scanner]") +{ + const bx::StringView input("abc def"); + + bx::Scanner sc(input); + + REQUIRE(input == sc.getTail() ); + + sc.accept(bx::Scanner::Class::Word); + + REQUIRE(" def" == sc.getTail() ); + REQUIRE(sc.getTail().getPtr() == input.getPtr() + 3); + REQUIRE(sc.getTail().getTerm() == input.getTerm() ); + + sc.acceptAll(); + + REQUIRE(sc.getTail().isEmpty() ); +} diff --git a/tests/sort_test.cpp b/tests/sort_test.cpp index 3629716..be63b11 100644 --- a/tests/sort_test.cpp +++ b/tests/sort_test.cpp @@ -181,3 +181,199 @@ TEST_CASE("ComparisonFn", "[sort]") compareTest< float>(-13.89f, 1389.0f); compareTest(-13.89f, 1389.0f); } + +#include +#include +#include +#include + +namespace +{ + struct Item + { + uint32_t key; + uint32_t seq; + }; + + bool byKey(const Item& _a, const Item& _b) + { + return _a.key < _b.key; + } + + tinystl::vector makeItems(uint32_t _num, uint32_t _numKeys) + { + tinystl::vector items; + uint32_t rng = 12345; + for (uint32_t ii = 0; ii < _num; ++ii) + { + rng = rng*1664525 + 1013904223; + const Item item = + { + .key = (rng >> 16) % _numKeys, + .seq = ii, + }; + + items.push_back(item); + } + + return items; + } + + void checkSortedAndStable(tinystl::vector& _items) + { + bx::DefaultAllocator allocator; + + const uint32_t num = uint32_t(_items.size() ); + bx::stableSort(&allocator, _items, byKey); + + REQUIRE(num == _items.size() ); + + for (uint32_t ii = 1; ii < num; ++ii) + { + REQUIRE(_items[ii-1].key <= _items[ii].key); + + if (_items[ii-1].key == _items[ii].key) + { + REQUIRE(_items[ii-1].seq < _items[ii].seq); + } + } + } + +} // namespace + +TEST_CASE("insertionSort", "[sort]") +{ + Item items[] = + { + { 3, 0 }, { 1, 1 }, { 3, 2 }, { 0, 3 }, { 1, 4 }, { 3, 5 }, + }; + + bx::insertionSort(items, BX_COUNTOF(items), byKey); + + const uint32_t expectedKey[] = { 0, 1, 1, 3, 3, 3 }; + const uint32_t expectedSeq[] = { 3, 1, 4, 0, 2, 5 }; + for (uint32_t ii = 0; ii < BX_COUNTOF(items); ++ii) + { + REQUIRE(expectedKey[ii] == items[ii].key); + REQUIRE(expectedSeq[ii] == items[ii].seq); + } + + bx::insertionSort(items, 0, byKey); + bx::insertionSort(items, 1, byKey); + REQUIRE(0 == items[0].key); +} + +TEST_CASE("stableSort is stable across the insertion/merge threshold", "[sort]") +{ + const uint32_t sizes[] = { 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000 }; + + for (uint32_t ii = 0; ii < BX_COUNTOF(sizes); ++ii) + { + tinystl::vector items = makeItems(sizes[ii], 4); + checkSortedAndStable(items); + } +} + +TEST_CASE("stableSort handles degenerate orderings", "[sort]") +{ + bx::DefaultAllocator allocator; + + { + tinystl::vector items; + for (uint32_t ii = 0; ii < 100; ++ii) + { + const Item item = { ii, ii }; + items.push_back(item); + } + + checkSortedAndStable(items); + REQUIRE(0 == items[0].key); + REQUIRE(99 == items[99].key); + } + + { + tinystl::vector items; + for (uint32_t ii = 0; ii < 100; ++ii) + { + const Item item = { 100-ii, ii }; + items.push_back(item); + } + + bx::stableSort(&allocator, items, byKey); + for (uint32_t ii = 1; ii < 100; ++ii) + { + REQUIRE(items[ii-1].key < items[ii].key); + } + } + + { + tinystl::vector items; + for (uint32_t ii = 0; ii < 100; ++ii) + { + const Item item = { 7, ii }; + items.push_back(item); + } + + bx::stableSort(&allocator, items, byKey); + for (uint32_t ii = 0; ii < 100; ++ii) + { + REQUIRE(ii == items[ii].seq); + } + } +} + +TEST_CASE("stableSort moves elements by assignment", "[sort]") +{ + struct Named + { + uint32_t key; + tinystl::string name; + }; + + const tinystl::string prefix("a-long-name-that-will-not-fit-in-a-small-string-buffer-"); + + bx::DefaultAllocator allocator; + + tinystl::vector items; + for (uint32_t ii = 0; ii < 200; ++ii) + { + const char suffix[] = { char('0' + ii%10) }; + + Named item; + item.key = (200 - ii) % 10; + item.name = prefix; + item.name.append(suffix, suffix + 1); + + items.push_back(item); + } + + bx::stableSort(&allocator, items, [](const Named& _a, const Named& _b) { return _a.key < _b.key; }); + + REQUIRE(200 == items.size() ); + for (uint32_t ii = 1; ii < items.size(); ++ii) + { + REQUIRE(items[ii-1].key <= items[ii].key); + } + + for (uint32_t ii = 0; ii < items.size(); ++ii) + { + REQUIRE(prefix.size() + 1 == items[ii].name.size() ); + REQUIRE(0 == bx::memCmp(items[ii].name.c_str(), prefix.c_str(), prefix.size() ) ); + } +} + +TEST_CASE("stableSort handles empty and single-element containers", "[sort]") +{ + bx::DefaultAllocator allocator; + + tinystl::vector empty; + bx::stableSort(&allocator, empty, byKey); + REQUIRE(empty.empty() ); + + tinystl::vector one; + const Item item = { 42, 0 }; + one.push_back(item); + bx::stableSort(&allocator, one, byKey); + REQUIRE(1 == one.size() ); + REQUIRE(42 == one[0].key); +}