Added Class::Word, Class::SpaceHoriz, acceptToken and getTail to Scanner. Added LiteralParser. Added stableSort and insertionSort. (#432)

This commit is contained in:
Branimir Karadžić
2026-09-27 18:32:54 -07:00
committed by GitHub
parent 3d6e53107b
commit 58d36bd042
15 changed files with 1329 additions and 2 deletions

View File

@@ -0,0 +1,27 @@
/*
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
# error "Must be included from bx/literal-parser.h!"
#endif // BX_LITERAL_PARSER_H_HEADER_GUARD
namespace bx
{
inline bool LiteralParser::isOk() const
{
return m_ok;
}
inline const StringView& LiteralParser::getLiteral() const
{
return m_literal;
}
inline LiteralParser::Type LiteralParser::getType() const
{
return m_type;
}
} // namespace bx

View File

@@ -85,6 +85,31 @@ namespace bx
return getCursor();
}
inline StringView Scanner::acceptToken(const StringView& _str)
{
if (!hasPrefix(m_tail, _str) )
{
return getCursor();
}
const char last = _str.getTerm()[-1];
const char* after = m_tail.getPtr() + _str.getLength();
if ( (isAlphaNum(last) || '_' == last)
&& after < m_input.getTerm()
&& (isAlphaNum(*after) || '_' == *after) )
{
return getCursor();
}
return moveTo({ m_tail.getPtr(), _str.getLength() });
}
inline StringView Scanner::getTail() const
{
return m_tail;
}
inline StringView Scanner::accept(Class _class)
{
return moveTo({ m_tail.getPtr(), strFunc(_class).getPtr() });

View File

@@ -55,6 +55,119 @@ namespace bx
quickSort( (void*)_data, _num, sizeof(Ty), _fn);
}
constexpr uint32_t kStableSortInsertionRun = 32;
template<typename Ty, typename LessFn>
inline void insertionSort(Ty* _data, uint32_t _num, LessFn _less)
{
for (uint32_t ii = 1; ii < _num; ++ii)
{
Ty item = _data[ii];
uint32_t jj = ii;
for (; jj > 0 && _less(item, _data[jj-1]); --jj)
{
_data[jj] = _data[jj-1];
}
_data[jj] = item;
}
}
template<typename Ty, typename LessFn>
inline void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less)
{
if (2 > _num)
{
return;
}
for (uint32_t ii = 0; ii < _num; ii += kStableSortInsertionRun)
{
insertionSort(_data + ii, min(kStableSortInsertionRun, _num - ii), _less);
}
if (kStableSortInsertionRun >= _num)
{
return;
}
Ty* scratch = (Ty*)alloc(_allocator, _num*sizeof(Ty) );
for (uint32_t ii = 0; ii < _num; ++ii)
{
BX_PLACEMENT_NEW(&scratch[ii], Ty);
}
Ty* src = _data;
Ty* dst = scratch;
for (uint32_t width = kStableSortInsertionRun; width < _num; width <<= 1)
{
for (uint32_t ii = 0; ii < _num; ii += width<<1)
{
const uint32_t mid = min(ii + width, _num);
const uint32_t end = min(ii + (width<<1), _num);
uint32_t ll = ii;
uint32_t rr = mid;
uint32_t oo = ii;
while (ll < mid
&& rr < end)
{
if (_less(src[rr], src[ll]) )
{
dst[oo] = src[rr];
++rr;
}
else
{
dst[oo] = src[ll];
++ll;
}
++oo;
}
for (; ll < mid; ++ll, ++oo)
{
dst[oo] = src[ll];
}
for (; rr < end; ++rr, ++oo)
{
dst[oo] = src[rr];
}
}
swap(src, dst);
}
if (src != _data)
{
for (uint32_t ii = 0; ii < _num; ++ii)
{
_data[ii] = src[ii];
}
}
for (uint32_t ii = 0; ii < _num; ++ii)
{
scratch[ii].~Ty();
}
free(_allocator, scratch);
}
template<typename ContainerT, typename LessFn>
inline void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less)
{
if (!_container.empty() )
{
stableSort(_allocator, &_container[0], uint32_t(_container.size() ), _less);
}
}
template<typename Ty>
inline uint32_t unique(void* _data, uint32_t _num, uint32_t _stride, const ComparisonFn _fn)
{

139
include/bx/literal-parser.h Normal file
View File

@@ -0,0 +1,139 @@
/*
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
#define BX_LITERAL_PARSER_H_HEADER_GUARD
#include <bx/string.h>
namespace bx
{
/// Parses numeric literal at the beginning of a string.
///
/// Syntax that's accepted is selected by `Profile`, so that numeric literals of different
/// languages can be read by the same parser. Profiles for a few languages are provided by
/// `wgsl`, `glsl`, `hlsl`, `c`, `cpp`, and `msl`.
///
/// Parsing stops at the first character that can't be part of literal, and trailing input is
/// not an error. Literal that was matched is returned by `getLiteral`, and it's a view into
/// input string.
///
/// @attention LiteralParser doesn't own input string. Input string must outlive LiteralParser.
///
class LiteralParser
{
BX_CLASS(LiteralParser
, NO_DEFAULT_CTOR
, NO_COPY
);
public:
/// Type of literal that was matched.
///
enum Type : uint8_t
{
Unknown, //!< Input doesn't start with literal that profile accepts.
Binary, //!< Binary integer, `0b1010`.
Octal, //!< Octal integer, `0o777`.
Decimal, //!< Decimal integer, `42`.
Hexadecimal, //!< Hexadecimal integer, `0x2a`.
Float, //!< Floating point number, `1.5`, `1e3`, or `0x1.8p3`.
};
/// Syntax flags, see `Profile::flags`.
///
enum : uint32_t
{
Sign = 1u<<0, //!< Leading `+` or `-` is part of literal.
HexPrefix = 1u<<1, //!< Hexadecimal `0x` and `0X` prefix.
BinPrefix = 1u<<2, //!< Binary `0b` prefix.
OctPrefix = 1u<<3, //!< Octal `0o` prefix.
HexFloat = 1u<<4, //!< Hexadecimal float, `0x1.8p3`. It requires `.` or `p`.
LeadingDot = 1u<<5, //!< Float without integer part, `.5`.
TrailingDot = 1u<<6, //!< Float without fractional part, `2.`.
UpperExponent = 1u<<7, //!< `E` and `P` exponent, in addition to `e` and `p`.
PromoteFloatSuffix = 1u<<8, //!< Float suffix makes decimal integer a float, WGSL `1f`.
NoLeadingZero = 1u<<9, //!< Decimal integer may not have leading zero. WGSL rejects `0123`, but `00012.` is a valid float.
};
/// Set of suffixes that literal can end with.
///
struct Suffixes
{
const StringView* data; //!< Suffixes to match. Longest match wins.
uint16_t num; //!< Number of suffixes.
};
/// Numeric syntax that LiteralParser accepts.
///
struct Profile
{
uint32_t flags; //!< Syntax flags.
char separator; //!< Digit separator, or 0 if there is none. For example `_` or `'`.
Suffixes intSuffix; //!< Suffixes accepted on integer literal.
Suffixes floatSuffix; //!< Suffixes accepted on float literal.
};
/// Returns profile that accepts sign, binary, octal and hexadecimal prefix, and `_` digit
/// separator, with no suffixes.
///
static Profile defaults();
/// Returns profile for WGSL numeric literals.
///
static Profile wgsl();
/// Returns profile for GLSL numeric literals.
///
static Profile glsl();
/// Returns profile for HLSL numeric literals.
///
static Profile hlsl();
/// Returns profile for C numeric literals.
///
static Profile c();
/// Returns profile for C++ numeric literals.
///
static Profile cpp();
/// Returns profile for Metal Shading Language numeric literals.
///
static Profile msl();
/// Constructor. Parses literal at the beginning of `_str`.
///
/// @param[in] _str Input string to parse. It's not copied, and it must outlive
/// LiteralParser.
/// @param[in] _profile Numeric syntax to accept.
///
LiteralParser(const StringView& _str, const Profile& _profile = defaults() );
/// Returns true if literal was matched.
///
bool isOk() const;
/// Returns literal that was matched, or empty string view if there was no match.
///
const StringView& getLiteral() const;
/// Returns type of literal that was matched, or `Type::Unknown` if there was no match.
///
Type getType() const;
private:
StringView m_literal;
Type m_type;
bool m_ok;
};
} // namespace bx
#include "inline/literal-parser.inl"
#endif // BX_LITERAL_PARSER_H_HEADER_GUARD

View File

@@ -29,9 +29,11 @@ namespace bx
///
enum class Class : uint8_t
{
Space, //!< Run of whitespace characters.
Space, //!< Run of whitespace characters, line terminators included.
SpaceHoriz, //!< Run of horizontal whitespace as `isSpaceHoriz` reads it, so up to but not including a line feed.
NonSpace, //!< Run of non-whitespace characters.
Identifier, //!< Run of alphanumeric characters and `_`.
Identifier, //!< Identifier as `strIdentifier` reads one: alphanumeric characters and `_`, not starting with a digit.
Word, //!< Word as `strWord` reads one: run of alphanumeric characters and `_`, a leading digit included.
EndOfLine, //!< Everything up to, but not including, line terminator.
NewLine, //!< Everything up to, and including, line terminator.
};
@@ -107,6 +109,19 @@ namespace bx
///
StringView accept(const StringView& _str);
/// Accept `_str` if input at cursor starts with it and it does not run into a word.
///
/// Unlike `accept`, which is a plain prefix test, a token ending in a word
/// character must be followed by something that is not one, so a keyword does not
/// match the head of a longer word: `acceptToken("if")` leaves `ifx` alone. For a
/// token ending in punctuation the two behave the same.
///
/// @param[in] _str Token to match.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
StringView acceptToken(const StringView& _str);
/// Accept leading run of characters belonging to `_class`.
///
/// @param[in] _class Character class to match.
@@ -234,6 +249,15 @@ namespace bx
///
StringView getCursor(Cursor _which = Cursor::Current) const;
/// Returns input from cursor to end of input.
///
/// Where `getCursor` gives a position, this gives the range still to be scanned,
/// which is what another parser over the same text needs handed to it.
///
/// @returns String view from cursor to end of input.
///
StringView getTail() const;
/// Returns text between `_from` and current cursor position.
///
/// @param[in] _from Position to measure from, usually obtained by `getCursor`.

View File

@@ -6,6 +6,7 @@
#ifndef BX_SORT_H_HEADER_GUARD
#define BX_SORT_H_HEADER_GUARD
#include "allocator.h"
#include "bx.h"
#include "math.h"
#include "string.h"
@@ -74,6 +75,52 @@ namespace bx
template<typename Ty>
void quickSort(Ty* _data, uint32_t _num, const ComparisonFn _fn = compareAscending<Ty>);
/// Performs sort (Insertion Sort algorithm).
///
/// It's stable, it sorts in place, and it doesn't allocate. Elements move by assignment rather
/// than by byte-wise swap, so unlike `quickSort` element type doesn't have to be trivially
/// move assignable. It's quadratic, and it's meant for short arrays, or as a building block of
/// `stableSort`.
///
/// @param _data Pointer to array data.
/// @param _num Number of elements.
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
/// strict weak ordering.
///
template<typename Ty, typename LessFn>
void insertionSort(Ty* _data, uint32_t _num, LessFn _less);
/// Performs stable sort: elements that compare equal keep their relative order.
///
/// Insertion sort for short runs, and bottom-up merge for the rest, ping-ponging between input
/// and a single scratch buffer. Elements move by assignment rather than by byte-wise swap, so
/// unlike `quickSort` element type doesn't have to be trivially move assignable. An element
/// holding a small-string buffer, or anything else self-referential, is safe here and is not
/// safe there.
///
/// @param _allocator Allocator for scratch buffer. Nothing is allocated when there are few
/// enough elements for insertion sort alone.
/// @param _data Pointer to array data.
/// @param _num Number of elements.
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
/// strict weak ordering.
///
/// @attention Element type must be default constructible and copy assignable.
///
template<typename Ty, typename LessFn>
void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less);
/// Performs stable sort over contiguous container.
///
/// @param _allocator Allocator for scratch buffer.
/// @param _container Container with `empty`, `size` and `operator[]`, such as
/// `tinystl::vector`.
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
/// strict weak ordering.
///
template<typename ContainerT, typename LessFn>
void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less);
/// Performs reordering of duplicate elements in the array in the way that unique elements
/// are sorted to the front of array, and duplicates are after the return value index.
///

View File

@@ -420,6 +420,9 @@ namespace bx
/// Returns string view with whitespace characters trimmed from left.
StringView strLTrimSpace(const StringView& _str);
/// Returns string view with horizontal whitespace characters trimmed from left.
StringView strLTrimSpaceHoriz(const StringView& _str);
/// Returns string view with non-whitespace characters trimmed from left.
StringView strLTrimNonSpace(const StringView& _str);

View File

@@ -78,6 +78,7 @@ project "bx"
path.join(BX_DIR, "src/file.cpp"),
path.join(BX_DIR, "src/filepath.cpp"),
path.join(BX_DIR, "src/hash.cpp"),
path.join(BX_DIR, "src/literal-parser.cpp"),
path.join(BX_DIR, "src/math.cpp"),
path.join(BX_DIR, "src/mutex.cpp"),
path.join(BX_DIR, "src/os.cpp"),

View File

@@ -14,6 +14,7 @@
#include "file.cpp"
#include "filepath.cpp"
#include "hash.cpp"
#include "literal-parser.cpp"
#include "math.cpp"
#include "mutex.cpp"
#include "os.cpp"

381
src/literal-parser.cpp Normal file
View File

@@ -0,0 +1,381 @@
/*
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#include <bx/literal-parser.h>
#include <bx/scanner.h>
namespace bx
{
namespace
{
static constexpr StringView kWgslInt[] = { "i", "u", "li", "lu" };
static constexpr StringView kWgslFloat[] = { "f", "h", "lf" };
static constexpr StringView kGlslInt[] = { "u", "U", "lu", "LU" };
static constexpr StringView kGlslFloat[] = { "f", "F", "lf", "LF" };
static constexpr StringView kHlslInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "UL", "ull", "llu", "ULL", "LLU" };
static constexpr StringView kHlslFloat[] = { "f", "F", "h", "H" };
static constexpr StringView kCInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "ull", "llu", "ULL", "LLU" };
static constexpr StringView kCFloat[] = { "f", "F", "l", "L" };
static constexpr StringView kMslFloat[] = { "f", "F", "h", "H" };
using LiteralProfile = LiteralParser::Profile;
using LiteralSuffixes = LiteralParser::Suffixes;
using LiteralType = LiteralParser::Type;
char peekChar(Scanner& _scanner)
{
const StringView view = _scanner.peek();
return view.isEmpty() ? '\0' : *view.getPtr();
}
bool acceptDigits(Scanner& _scanner, CharTestFn _fn, char _separator)
{
bool ok = false;
for (char ch = peekChar(_scanner)
; _fn(ch) || (0 != _separator && _separator == ch)
; ch = peekChar(_scanner) )
{
ok |= _fn(ch);
_scanner.accept();
}
return ok;
}
bool acceptSuffix(Scanner& _scanner, const LiteralSuffixes& _suffixes)
{
int32_t best = -1;
int32_t bestLen = 0;
for (uint16_t ii = 0; ii < _suffixes.num; ++ii)
{
const int32_t len = _suffixes.data[ii].getLength();
if (len > bestLen
&& !_scanner.peek(_suffixes.data[ii]).isEmpty() )
{
best = ii;
bestLen = len;
}
}
if (0 <= best)
{
_scanner.accept(_suffixes.data[best]);
return true;
}
return false;
}
enum ExponentResult
{
ExponentNone,
ExponentOk,
ExponentBad,
};
ExponentResult acceptExponent(Scanner& _scanner, const LiteralProfile& _profile, CharTestFn _digits)
{
const char ch = peekChar(_scanner);
const bool isExponent = 'e' == ch
|| (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch)
;
if (!isExponent)
{
return ExponentNone;
}
_scanner.accept();
_scanner.accept('-', '+');
return acceptDigits(_scanner, _digits, _profile.separator) ? ExponentOk : ExponentBad;
}
LiteralType parseDecimalFloat(Scanner& _scanner, const LiteralProfile& _profile)
{
bool dot = false;
bool frac = false;
if (!_scanner.accept('.').isEmpty() )
{
dot = true;
frac = acceptDigits(_scanner, isNumeric, _profile.separator);
}
const ExponentResult exponent = acceptExponent(_scanner, _profile, isNumeric);
if (ExponentBad == exponent)
{
return LiteralType::Unknown;
}
const bool ok = frac
|| ExponentOk == exponent
|| (dot && 0 != (_profile.flags & LiteralParser::TrailingDot) )
;
if (!ok)
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.floatSuffix);
return LiteralType::Float;
}
LiteralType parseAfterInteger(Scanner& _scanner, const LiteralProfile& _profile)
{
const char ch = peekChar(_scanner);
const bool isExponent = 'e' == ch
|| (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch)
;
if ('.' == ch
|| isExponent)
{
return parseDecimalFloat(_scanner, _profile);
}
if (0 != (_profile.flags & LiteralParser::PromoteFloatSuffix)
&& acceptSuffix(_scanner, _profile.floatSuffix) )
{
return LiteralType::Float;
}
acceptSuffix(_scanner, _profile.intSuffix);
return LiteralType::Decimal;
}
LiteralType parseHex(Scanner& _scanner, const LiteralProfile& _profile)
{
const bool anyHex = acceptDigits(_scanner, isHexNum, _profile.separator);
bool isFloat = false;
bool anyHexFrac = false;
if (0 != (_profile.flags & LiteralParser::HexFloat) )
{
if (!_scanner.accept('.').isEmpty() )
{
isFloat = true;
anyHexFrac = acceptDigits(_scanner, isHexNum, _profile.separator);
}
const char ch = peekChar(_scanner);
if ('p' == ch
|| 'P' == ch)
{
isFloat = true;
_scanner.accept();
_scanner.accept('-', '+');
if (!acceptDigits(_scanner, isNumeric, _profile.separator) )
{
return LiteralType::Unknown;
}
}
}
if (isFloat)
{
// `0x.p2` has no significand digits.
if (!anyHex
&& !anyHexFrac)
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.floatSuffix);
return LiteralType::Float;
}
// `0x` with no digits.
if (!anyHex)
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.intSuffix);
return LiteralType::Hexadecimal;
}
LiteralType parseLiteral(Scanner& _scanner, const LiteralProfile& _profile)
{
if (0 != (_profile.flags & LiteralParser::Sign) )
{
_scanner.accept('-', '+');
}
if (!_scanner.accept('0').isEmpty() )
{
if (0 != (_profile.flags & LiteralParser::BinPrefix)
&& !_scanner.accept('b').isEmpty() )
{
if (!acceptDigits(_scanner, isBinNum, _profile.separator) )
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.intSuffix);
return LiteralType::Binary;
}
if (0 != (_profile.flags & LiteralParser::OctPrefix)
&& !_scanner.accept('o').isEmpty() )
{
if (!acceptDigits(_scanner, isOctNum, _profile.separator) )
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.intSuffix);
return LiteralType::Octal;
}
if (0 != (_profile.flags & LiteralParser::HexPrefix)
&& !_scanner.accept('x', 'X').isEmpty() )
{
return parseHex(_scanner, _profile);
}
const bool leadingZeros = acceptDigits(_scanner, isNumeric, _profile.separator);
const LiteralType type = parseAfterInteger(_scanner, _profile);
if (leadingZeros
&& LiteralType::Decimal == type
&& 0 != (_profile.flags & LiteralParser::NoLeadingZero) )
{
return LiteralType::Unknown;
}
return type;
}
if (isNumeric(peekChar(_scanner) ) )
{
acceptDigits(_scanner, isNumeric, _profile.separator);
return parseAfterInteger(_scanner, _profile);
}
if (0 != (_profile.flags & LiteralParser::LeadingDot)
&& '.' == peekChar(_scanner) )
{
_scanner.accept('.');
// `.` must be followed by a digit.
if (!acceptDigits(_scanner, isNumeric, _profile.separator) )
{
return LiteralType::Unknown;
}
if (ExponentBad == acceptExponent(_scanner, _profile, isNumeric) )
{
return LiteralType::Unknown;
}
acceptSuffix(_scanner, _profile.floatSuffix);
return LiteralType::Float;
}
return LiteralType::Unknown;
}
} // namespace
LiteralParser::Profile LiteralParser::defaults()
{
return
{
Sign | BinPrefix | OctPrefix | HexPrefix,
'_',
{ NULL, 0 },
{ NULL, 0 },
};
}
LiteralParser::Profile LiteralParser::wgsl()
{
return
{
HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix | NoLeadingZero,
0,
{ kWgslInt, uint16_t(BX_COUNTOF(kWgslInt) ) },
{ kWgslFloat, uint16_t(BX_COUNTOF(kWgslFloat) ) },
};
}
LiteralParser::Profile LiteralParser::glsl()
{
return
{
HexPrefix | LeadingDot | TrailingDot | UpperExponent,
0,
{ kGlslInt, uint16_t(BX_COUNTOF(kGlslInt) ) },
{ kGlslFloat, uint16_t(BX_COUNTOF(kGlslFloat) ) },
};
}
LiteralParser::Profile LiteralParser::hlsl()
{
return
{
HexPrefix | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix,
0,
{ kHlslInt, uint16_t(BX_COUNTOF(kHlslInt) ) },
{ kHlslFloat, uint16_t(BX_COUNTOF(kHlslFloat) ) },
};
}
LiteralParser::Profile LiteralParser::c()
{
return
{
HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent,
0,
{ kCInt, uint16_t(BX_COUNTOF(kCInt) ) },
{ kCFloat, uint16_t(BX_COUNTOF(kCFloat) ) },
};
}
LiteralParser::Profile LiteralParser::cpp()
{
Profile profile = c();
profile.flags |= BinPrefix;
profile.separator = '\'';
return profile;
}
LiteralParser::Profile LiteralParser::msl()
{
Profile profile = cpp();
profile.floatSuffix = { kMslFloat, uint16_t(BX_COUNTOF(kMslFloat) ) };
return profile;
}
LiteralParser::LiteralParser(const StringView& _str, const Profile& _profile)
: m_type(Type::Unknown)
, m_ok(false)
{
Scanner scanner(_str);
const StringView start = scanner.getCursor();
m_type = parseLiteral(scanner, _profile);
m_ok = Type::Unknown != m_type;
m_literal = m_ok
? scanner.between(start)
: StringView()
;
}
} // namespace bx

View File

@@ -90,6 +90,9 @@ namespace bx
case Class::NonSpace:
return strLTrimNonSpace(m_tail);
case Class::SpaceHoriz:
return strLTrimSpaceHoriz(m_tail);
case Class::Identifier:
{
const StringView word = strIdentifier(m_tail);
@@ -99,6 +102,15 @@ namespace bx
;
}
case Class::Word:
{
const StringView word = strWord(m_tail);
return word.isEmpty()
? m_tail
: StringView(word.getTerm(), m_tail.getTerm() )
;
}
case Class::EndOfLine:
return strFindEol(m_tail);

View File

@@ -410,6 +410,19 @@ namespace bx
return StringView(_str.getTerm(), _str.getTerm() );
}
StringView strLTrimSpaceHoriz(const StringView& _str)
{
for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr)
{
if (!isSpaceHoriz(*ptr) )
{
return StringView(ptr, term);
}
}
return StringView(_str.getTerm(), _str.getTerm() );
}
StringView strLTrimNonSpace(const StringView& _str)
{
for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr)

View File

@@ -0,0 +1,236 @@
/*
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#include "test.h"
#include <bx/literal-parser.h>
bool isNumber(bx::StringView _str)
{
const char* str = _str.getPtr();
int32_t len = _str.getLength();
const char* end = str + len;
const char* p = str;
if (p < end && (*p == '-' || *p == '+'))
{
p++;
}
bool has_digit = false;
while (p < end && isdigit((unsigned char)*p))
{
has_digit = true;
p++;
}
if (!has_digit) return false;
if (p < end && *p == '.')
{
p++;
has_digit = false;
while (p < end && isdigit((unsigned char)*p))
{
has_digit = true;
p++;
}
if (!has_digit) return false;
}
if (p < end && (*p == 'e' || *p == 'E'))
{
p++;
if (p < end && (*p == '-' || *p == '+'))
{
p++;
}
has_digit = false;
while (p < end && isdigit((unsigned char)*p))
{
has_digit = true;
p++;
}
if (!has_digit) return false;
}
return p == end;
}
bool testNumber(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown)
{
bx::LiteralParser p(_input);
const bx::StringView literal = p.getLiteral();
bool ok = true
&& p.isOk()
&& _type == p.getType()
&& bx::isEqual(literal, _output)
;
if (!ok
&& _type != bx::LiteralParser::Type::Unknown)
{
DBG("%d: %S, expected %d: %S", p.getType(), &literal, _type, &_output);
}
BX_TRACE("is_number %d, %d <- %S"
, isNumber(literal)
, ok
, &literal
);
return ok;
}
bool testNumber(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown)
{
return testNumber(_inputAndOutput, _inputAndOutput, _type);
}
TEST_CASE("LiteralParser", "[parser]")
{
// error
REQUIRE(!testNumber("") );
REQUIRE(!testNumber(".") );
REQUIRE(!testNumber("abvgd") );
REQUIRE(!testNumber("-e0") );
REQUIRE(!testNumber("-e-89") );
REQUIRE(!testNumber("0b") );
REQUIRE(!testNumber("0o") );
REQUIRE(!testNumber("0x") );
REQUIRE(!testNumber("0e") );
REQUIRE(!testNumber("-0.e-") );
REQUIRE(!testNumber("1.38913e-") );
// decimal
REQUIRE(testNumber("0", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("00", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("000", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0000", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("00_00", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0_0_0_0", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0-", "0", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0-e0", "0", bx::LiteralParser::Type::Decimal) );
// binary
REQUIRE(testNumber("0b0001001110001001", bx::LiteralParser::Type::Binary) );
REQUIRE(testNumber("0b0001_0011_1000_1001", bx::LiteralParser::Type::Binary) );
REQUIRE(testNumber("0b0;", "0b0", bx::LiteralParser::Type::Binary) );
REQUIRE(testNumber("0b0+", "0b0", bx::LiteralParser::Type::Binary) );
// octal
REQUIRE(testNumber("02555", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0o2555", bx::LiteralParser::Type::Octal) );
REQUIRE(testNumber("0o25_55", bx::LiteralParser::Type::Octal) );
// hex
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0x0;", "0x0", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0x0+", "0x0", bx::LiteralParser::Type::Hexadecimal) );
// integer
REQUIRE(testNumber("1389", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("13_89", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) );
REQUIRE(testNumber("0+", "0", bx::LiteralParser::Type::Decimal) );
// floating point
REQUIRE(testNumber("0.1389", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("-0.1389", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("+0.1389", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("0.e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("-0.e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("+0.e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("138e-9", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("1.38913e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("1.389e+001", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("1.389e-002", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("1.389e-002;", "1.389e-002", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("-0.e-89;", "-0.e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("-0.e-89+", "-0.e-89", bx::LiteralParser::Type::Float) );
REQUIRE(testNumber("-0.e-89e", "-0.e-89", bx::LiteralParser::Type::Float) );
// hex with letters (lower/upper/mixed) and digit separators
REQUIRE(testNumber("0xabcdef", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xABCDEF", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xDeadC0de", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xff", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0x13_89", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xFF_FF", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xdead;", "0xdead", bx::LiteralParser::Type::Hexadecimal) );
REQUIRE(testNumber("0xC0DE-", "0xC0DE", bx::LiteralParser::Type::Hexadecimal) );
// more errors: a lone sign or a leading separator is not a number
REQUIRE(!testNumber("+") );
REQUIRE(!testNumber("-") );
REQUIRE(!testNumber("_5") );
REQUIRE(!testNumber("0xg") );
}
static bool testWgsl(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type)
{
bx::LiteralParser p(_input, bx::LiteralParser::wgsl() );
return p.isOk()
&& _type == p.getType()
&& bx::isEqual(p.getLiteral(), _output)
;
}
static bool testWgsl(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type)
{
return testWgsl(_inputAndOutput, _inputAndOutput, _type);
}
TEST_CASE("LiteralParser WGSL profile", "[parser]")
{
typedef bx::LiteralParser LP;
// integers (+ WGSL i/u/li/lu suffixes) and hex
REQUIRE(testWgsl("0", LP::Decimal) );
REQUIRE(testWgsl("42", LP::Decimal) );
REQUIRE(testWgsl("1u", LP::Decimal) );
REQUIRE(testWgsl("1i", LP::Decimal) );
REQUIRE(testWgsl("1li", LP::Decimal) );
REQUIRE(testWgsl("1lu", LP::Decimal) );
REQUIRE(testWgsl("0xFF", LP::Hexadecimal) );
REQUIRE(testWgsl("0xffu", LP::Hexadecimal) );
// floats: fraction/exponent, edge dots, suffixes, hex float
REQUIRE(testWgsl("1.0", LP::Float) );
REQUIRE(testWgsl(".5", LP::Float) );
REQUIRE(testWgsl("2.", LP::Float) );
REQUIRE(testWgsl("1e5", LP::Float) );
REQUIRE(testWgsl("3.14e-2", LP::Float) );
REQUIRE(testWgsl("1.5f", LP::Float) );
REQUIRE(testWgsl("1f", LP::Float) ); // bare integer + float suffix -> float
REQUIRE(testWgsl("1h", LP::Float) );
REQUIRE(testWgsl("1lf", LP::Float) );
REQUIRE(testWgsl("0x1p-3", LP::Float) );
REQUIRE(testWgsl("0x1.8p3", LP::Float) );
// prefix match: trailing input is not part of the literal
REQUIRE(testWgsl("42;", "42", LP::Decimal) );
REQUIRE(testWgsl("0xFFu-", "0xFFu", LP::Hexadecimal) );
REQUIRE(testWgsl("1.x", "1.", LP::Float) ); // "1." is a float (trailing dot), then ".x"
// WGSL has no sign, no 0b/0o prefixes, no '_' separators
REQUIRE(!bx::LiteralParser("-5", LP::wgsl() ).isOk() ); // sign is a separate token
REQUIRE(testWgsl("0b1", "0", LP::Decimal) ); // "0" then "b1"
REQUIRE(testWgsl("5_5", "5", LP::Decimal) ); // "5" then "_5"
}

View File

@@ -390,3 +390,112 @@ TEST_CASE("LineReader", "[scanner][string]")
REQUIRE(second.getPtr() == input.getPtr() + 4);
}
}
TEST_CASE("Scanner.class Word", "[scanner]")
{
{
bx::Scanner sc("2d, f32");
REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) );
REQUIRE("," == sc.accept(',') );
}
{
bx::Scanner sc("2d");
REQUIRE(sc.accept(bx::Scanner::Class::Identifier).isEmpty() );
REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) );
}
{
bx::Scanner sc("_leading and9digits");
REQUIRE("_leading" == sc.accept(bx::Scanner::Class::Word) );
REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) );
REQUIRE("and9digits" == sc.accept(bx::Scanner::Class::Word) );
REQUIRE(sc.isDone() );
}
}
TEST_CASE("Scanner.class SpaceHoriz", "[scanner]")
{
{
bx::Scanner horiz(" \t\nx");
REQUIRE(" \t" == horiz.accept(bx::Scanner::Class::SpaceHoriz) );
REQUIRE("\n" == horiz.accept('\n') );
bx::Scanner space(" \t\nx");
REQUIRE(" \t\n" == space.accept(bx::Scanner::Class::Space) );
}
{
bx::Scanner sc("x");
REQUIRE(sc.accept(bx::Scanner::Class::SpaceHoriz).isEmpty() );
REQUIRE("x" == sc.accept('x') );
}
{
bx::Scanner sc("a \r\nb");
REQUIRE("a" == sc.accept(bx::Scanner::Class::Word) );
REQUIRE(" \r" == sc.accept(bx::Scanner::Class::SpaceHoriz) );
REQUIRE("\n" == sc.accept('\n') );
REQUIRE("b" == sc.accept(bx::Scanner::Class::Word) );
}
}
TEST_CASE("Scanner.acceptToken", "[scanner]")
{
{
bx::Scanner bare("ifx");
REQUIRE("if" == bare.accept(bx::StringView("if") ) );
bx::Scanner token("ifx");
REQUIRE(token.acceptToken(bx::StringView("if") ).isEmpty() );
REQUIRE("ifx" == token.accept(bx::Scanner::Class::Word) );
}
{
bx::Scanner sc("if (x)");
REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) );
REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) );
REQUIRE("(" == sc.acceptToken(bx::StringView("(") ) );
}
{
bx::Scanner sc("->x");
REQUIRE("->" == sc.acceptToken(bx::StringView("->") ) );
REQUIRE("x" == sc.accept(bx::Scanner::Class::Word) );
}
{
bx::Scanner sc("if");
REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) );
REQUIRE(sc.isDone() );
}
}
TEST_CASE("Scanner.getTail", "[scanner]")
{
const bx::StringView input("abc def");
bx::Scanner sc(input);
REQUIRE(input == sc.getTail() );
sc.accept(bx::Scanner::Class::Word);
REQUIRE(" def" == sc.getTail() );
REQUIRE(sc.getTail().getPtr() == input.getPtr() + 3);
REQUIRE(sc.getTail().getTerm() == input.getTerm() );
sc.acceptAll();
REQUIRE(sc.getTail().isEmpty() );
}

View File

@@ -181,3 +181,199 @@ TEST_CASE("ComparisonFn", "[sort]")
compareTest< float>(-13.89f, 1389.0f);
compareTest<double>(-13.89f, 1389.0f);
}
#include <bx/allocator.h>
#include <tinystl/allocator.h>
#include <tinystl/string.h>
#include <tinystl/vector.h>
namespace
{
struct Item
{
uint32_t key;
uint32_t seq;
};
bool byKey(const Item& _a, const Item& _b)
{
return _a.key < _b.key;
}
tinystl::vector<Item> makeItems(uint32_t _num, uint32_t _numKeys)
{
tinystl::vector<Item> items;
uint32_t rng = 12345;
for (uint32_t ii = 0; ii < _num; ++ii)
{
rng = rng*1664525 + 1013904223;
const Item item =
{
.key = (rng >> 16) % _numKeys,
.seq = ii,
};
items.push_back(item);
}
return items;
}
void checkSortedAndStable(tinystl::vector<Item>& _items)
{
bx::DefaultAllocator allocator;
const uint32_t num = uint32_t(_items.size() );
bx::stableSort(&allocator, _items, byKey);
REQUIRE(num == _items.size() );
for (uint32_t ii = 1; ii < num; ++ii)
{
REQUIRE(_items[ii-1].key <= _items[ii].key);
if (_items[ii-1].key == _items[ii].key)
{
REQUIRE(_items[ii-1].seq < _items[ii].seq);
}
}
}
} // namespace
TEST_CASE("insertionSort", "[sort]")
{
Item items[] =
{
{ 3, 0 }, { 1, 1 }, { 3, 2 }, { 0, 3 }, { 1, 4 }, { 3, 5 },
};
bx::insertionSort(items, BX_COUNTOF(items), byKey);
const uint32_t expectedKey[] = { 0, 1, 1, 3, 3, 3 };
const uint32_t expectedSeq[] = { 3, 1, 4, 0, 2, 5 };
for (uint32_t ii = 0; ii < BX_COUNTOF(items); ++ii)
{
REQUIRE(expectedKey[ii] == items[ii].key);
REQUIRE(expectedSeq[ii] == items[ii].seq);
}
bx::insertionSort(items, 0, byKey);
bx::insertionSort(items, 1, byKey);
REQUIRE(0 == items[0].key);
}
TEST_CASE("stableSort is stable across the insertion/merge threshold", "[sort]")
{
const uint32_t sizes[] = { 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000 };
for (uint32_t ii = 0; ii < BX_COUNTOF(sizes); ++ii)
{
tinystl::vector<Item> items = makeItems(sizes[ii], 4);
checkSortedAndStable(items);
}
}
TEST_CASE("stableSort handles degenerate orderings", "[sort]")
{
bx::DefaultAllocator allocator;
{
tinystl::vector<Item> items;
for (uint32_t ii = 0; ii < 100; ++ii)
{
const Item item = { ii, ii };
items.push_back(item);
}
checkSortedAndStable(items);
REQUIRE(0 == items[0].key);
REQUIRE(99 == items[99].key);
}
{
tinystl::vector<Item> items;
for (uint32_t ii = 0; ii < 100; ++ii)
{
const Item item = { 100-ii, ii };
items.push_back(item);
}
bx::stableSort(&allocator, items, byKey);
for (uint32_t ii = 1; ii < 100; ++ii)
{
REQUIRE(items[ii-1].key < items[ii].key);
}
}
{
tinystl::vector<Item> items;
for (uint32_t ii = 0; ii < 100; ++ii)
{
const Item item = { 7, ii };
items.push_back(item);
}
bx::stableSort(&allocator, items, byKey);
for (uint32_t ii = 0; ii < 100; ++ii)
{
REQUIRE(ii == items[ii].seq);
}
}
}
TEST_CASE("stableSort moves elements by assignment", "[sort]")
{
struct Named
{
uint32_t key;
tinystl::string name;
};
const tinystl::string prefix("a-long-name-that-will-not-fit-in-a-small-string-buffer-");
bx::DefaultAllocator allocator;
tinystl::vector<Named> items;
for (uint32_t ii = 0; ii < 200; ++ii)
{
const char suffix[] = { char('0' + ii%10) };
Named item;
item.key = (200 - ii) % 10;
item.name = prefix;
item.name.append(suffix, suffix + 1);
items.push_back(item);
}
bx::stableSort(&allocator, items, [](const Named& _a, const Named& _b) { return _a.key < _b.key; });
REQUIRE(200 == items.size() );
for (uint32_t ii = 1; ii < items.size(); ++ii)
{
REQUIRE(items[ii-1].key <= items[ii].key);
}
for (uint32_t ii = 0; ii < items.size(); ++ii)
{
REQUIRE(prefix.size() + 1 == items[ii].name.size() );
REQUIRE(0 == bx::memCmp(items[ii].name.c_str(), prefix.c_str(), prefix.size() ) );
}
}
TEST_CASE("stableSort handles empty and single-element containers", "[sort]")
{
bx::DefaultAllocator allocator;
tinystl::vector<Item> empty;
bx::stableSort(&allocator, empty, byKey);
REQUIRE(empty.empty() );
tinystl::vector<Item> one;
const Item item = { 42, 0 };
one.push_back(item);
bx::stableSort(&allocator, one, byKey);
REQUIRE(1 == one.size() );
REQUIRE(42 == one[0].key);
}