mirror of
https://github.com/bkaradzic/bx.git
synced 2026-10-22 16:51:01 +00:00
Added Class::Word, Class::SpaceHoriz, acceptToken and getTail to Scanner. Added LiteralParser. Added stableSort and insertionSort. (#432)
This commit is contained in:
committed by
GitHub
parent
3d6e53107b
commit
58d36bd042
27
include/bx/inline/literal-parser.inl
Normal file
27
include/bx/inline/literal-parser.inl
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
|
||||
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
|
||||
# error "Must be included from bx/literal-parser.h!"
|
||||
#endif // BX_LITERAL_PARSER_H_HEADER_GUARD
|
||||
|
||||
namespace bx
|
||||
{
|
||||
inline bool LiteralParser::isOk() const
|
||||
{
|
||||
return m_ok;
|
||||
}
|
||||
|
||||
inline const StringView& LiteralParser::getLiteral() const
|
||||
{
|
||||
return m_literal;
|
||||
}
|
||||
|
||||
inline LiteralParser::Type LiteralParser::getType() const
|
||||
{
|
||||
return m_type;
|
||||
}
|
||||
|
||||
} // namespace bx
|
||||
@@ -85,6 +85,31 @@ namespace bx
|
||||
return getCursor();
|
||||
}
|
||||
|
||||
inline StringView Scanner::acceptToken(const StringView& _str)
|
||||
{
|
||||
if (!hasPrefix(m_tail, _str) )
|
||||
{
|
||||
return getCursor();
|
||||
}
|
||||
|
||||
const char last = _str.getTerm()[-1];
|
||||
const char* after = m_tail.getPtr() + _str.getLength();
|
||||
|
||||
if ( (isAlphaNum(last) || '_' == last)
|
||||
&& after < m_input.getTerm()
|
||||
&& (isAlphaNum(*after) || '_' == *after) )
|
||||
{
|
||||
return getCursor();
|
||||
}
|
||||
|
||||
return moveTo({ m_tail.getPtr(), _str.getLength() });
|
||||
}
|
||||
|
||||
inline StringView Scanner::getTail() const
|
||||
{
|
||||
return m_tail;
|
||||
}
|
||||
|
||||
inline StringView Scanner::accept(Class _class)
|
||||
{
|
||||
return moveTo({ m_tail.getPtr(), strFunc(_class).getPtr() });
|
||||
|
||||
@@ -55,6 +55,119 @@ namespace bx
|
||||
quickSort( (void*)_data, _num, sizeof(Ty), _fn);
|
||||
}
|
||||
|
||||
constexpr uint32_t kStableSortInsertionRun = 32;
|
||||
|
||||
template<typename Ty, typename LessFn>
|
||||
inline void insertionSort(Ty* _data, uint32_t _num, LessFn _less)
|
||||
{
|
||||
for (uint32_t ii = 1; ii < _num; ++ii)
|
||||
{
|
||||
Ty item = _data[ii];
|
||||
|
||||
uint32_t jj = ii;
|
||||
for (; jj > 0 && _less(item, _data[jj-1]); --jj)
|
||||
{
|
||||
_data[jj] = _data[jj-1];
|
||||
}
|
||||
|
||||
_data[jj] = item;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Ty, typename LessFn>
|
||||
inline void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less)
|
||||
{
|
||||
if (2 > _num)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint32_t ii = 0; ii < _num; ii += kStableSortInsertionRun)
|
||||
{
|
||||
insertionSort(_data + ii, min(kStableSortInsertionRun, _num - ii), _less);
|
||||
}
|
||||
|
||||
if (kStableSortInsertionRun >= _num)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Ty* scratch = (Ty*)alloc(_allocator, _num*sizeof(Ty) );
|
||||
for (uint32_t ii = 0; ii < _num; ++ii)
|
||||
{
|
||||
BX_PLACEMENT_NEW(&scratch[ii], Ty);
|
||||
}
|
||||
|
||||
Ty* src = _data;
|
||||
Ty* dst = scratch;
|
||||
|
||||
for (uint32_t width = kStableSortInsertionRun; width < _num; width <<= 1)
|
||||
{
|
||||
for (uint32_t ii = 0; ii < _num; ii += width<<1)
|
||||
{
|
||||
const uint32_t mid = min(ii + width, _num);
|
||||
const uint32_t end = min(ii + (width<<1), _num);
|
||||
|
||||
uint32_t ll = ii;
|
||||
uint32_t rr = mid;
|
||||
uint32_t oo = ii;
|
||||
|
||||
while (ll < mid
|
||||
&& rr < end)
|
||||
{
|
||||
if (_less(src[rr], src[ll]) )
|
||||
{
|
||||
dst[oo] = src[rr];
|
||||
++rr;
|
||||
}
|
||||
else
|
||||
{
|
||||
dst[oo] = src[ll];
|
||||
++ll;
|
||||
}
|
||||
|
||||
++oo;
|
||||
}
|
||||
|
||||
for (; ll < mid; ++ll, ++oo)
|
||||
{
|
||||
dst[oo] = src[ll];
|
||||
}
|
||||
|
||||
for (; rr < end; ++rr, ++oo)
|
||||
{
|
||||
dst[oo] = src[rr];
|
||||
}
|
||||
}
|
||||
|
||||
swap(src, dst);
|
||||
}
|
||||
|
||||
if (src != _data)
|
||||
{
|
||||
for (uint32_t ii = 0; ii < _num; ++ii)
|
||||
{
|
||||
_data[ii] = src[ii];
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t ii = 0; ii < _num; ++ii)
|
||||
{
|
||||
scratch[ii].~Ty();
|
||||
}
|
||||
|
||||
free(_allocator, scratch);
|
||||
}
|
||||
|
||||
template<typename ContainerT, typename LessFn>
|
||||
inline void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less)
|
||||
{
|
||||
if (!_container.empty() )
|
||||
{
|
||||
stableSort(_allocator, &_container[0], uint32_t(_container.size() ), _less);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Ty>
|
||||
inline uint32_t unique(void* _data, uint32_t _num, uint32_t _stride, const ComparisonFn _fn)
|
||||
{
|
||||
|
||||
139
include/bx/literal-parser.h
Normal file
139
include/bx/literal-parser.h
Normal file
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
|
||||
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
|
||||
#define BX_LITERAL_PARSER_H_HEADER_GUARD
|
||||
|
||||
#include <bx/string.h>
|
||||
|
||||
namespace bx
|
||||
{
|
||||
/// Parses numeric literal at the beginning of a string.
|
||||
///
|
||||
/// Syntax that's accepted is selected by `Profile`, so that numeric literals of different
|
||||
/// languages can be read by the same parser. Profiles for a few languages are provided by
|
||||
/// `wgsl`, `glsl`, `hlsl`, `c`, `cpp`, and `msl`.
|
||||
///
|
||||
/// Parsing stops at the first character that can't be part of literal, and trailing input is
|
||||
/// not an error. Literal that was matched is returned by `getLiteral`, and it's a view into
|
||||
/// input string.
|
||||
///
|
||||
/// @attention LiteralParser doesn't own input string. Input string must outlive LiteralParser.
|
||||
///
|
||||
class LiteralParser
|
||||
{
|
||||
BX_CLASS(LiteralParser
|
||||
, NO_DEFAULT_CTOR
|
||||
, NO_COPY
|
||||
);
|
||||
|
||||
public:
|
||||
/// Type of literal that was matched.
|
||||
///
|
||||
enum Type : uint8_t
|
||||
{
|
||||
Unknown, //!< Input doesn't start with literal that profile accepts.
|
||||
|
||||
Binary, //!< Binary integer, `0b1010`.
|
||||
Octal, //!< Octal integer, `0o777`.
|
||||
Decimal, //!< Decimal integer, `42`.
|
||||
Hexadecimal, //!< Hexadecimal integer, `0x2a`.
|
||||
Float, //!< Floating point number, `1.5`, `1e3`, or `0x1.8p3`.
|
||||
};
|
||||
|
||||
/// Syntax flags, see `Profile::flags`.
|
||||
///
|
||||
enum : uint32_t
|
||||
{
|
||||
Sign = 1u<<0, //!< Leading `+` or `-` is part of literal.
|
||||
HexPrefix = 1u<<1, //!< Hexadecimal `0x` and `0X` prefix.
|
||||
BinPrefix = 1u<<2, //!< Binary `0b` prefix.
|
||||
OctPrefix = 1u<<3, //!< Octal `0o` prefix.
|
||||
HexFloat = 1u<<4, //!< Hexadecimal float, `0x1.8p3`. It requires `.` or `p`.
|
||||
LeadingDot = 1u<<5, //!< Float without integer part, `.5`.
|
||||
TrailingDot = 1u<<6, //!< Float without fractional part, `2.`.
|
||||
UpperExponent = 1u<<7, //!< `E` and `P` exponent, in addition to `e` and `p`.
|
||||
PromoteFloatSuffix = 1u<<8, //!< Float suffix makes decimal integer a float, WGSL `1f`.
|
||||
NoLeadingZero = 1u<<9, //!< Decimal integer may not have leading zero. WGSL rejects `0123`, but `00012.` is a valid float.
|
||||
};
|
||||
|
||||
/// Set of suffixes that literal can end with.
|
||||
///
|
||||
struct Suffixes
|
||||
{
|
||||
const StringView* data; //!< Suffixes to match. Longest match wins.
|
||||
uint16_t num; //!< Number of suffixes.
|
||||
};
|
||||
|
||||
/// Numeric syntax that LiteralParser accepts.
|
||||
///
|
||||
struct Profile
|
||||
{
|
||||
uint32_t flags; //!< Syntax flags.
|
||||
char separator; //!< Digit separator, or 0 if there is none. For example `_` or `'`.
|
||||
Suffixes intSuffix; //!< Suffixes accepted on integer literal.
|
||||
Suffixes floatSuffix; //!< Suffixes accepted on float literal.
|
||||
};
|
||||
|
||||
/// Returns profile that accepts sign, binary, octal and hexadecimal prefix, and `_` digit
|
||||
/// separator, with no suffixes.
|
||||
///
|
||||
static Profile defaults();
|
||||
|
||||
/// Returns profile for WGSL numeric literals.
|
||||
///
|
||||
static Profile wgsl();
|
||||
|
||||
/// Returns profile for GLSL numeric literals.
|
||||
///
|
||||
static Profile glsl();
|
||||
|
||||
/// Returns profile for HLSL numeric literals.
|
||||
///
|
||||
static Profile hlsl();
|
||||
|
||||
/// Returns profile for C numeric literals.
|
||||
///
|
||||
static Profile c();
|
||||
|
||||
/// Returns profile for C++ numeric literals.
|
||||
///
|
||||
static Profile cpp();
|
||||
|
||||
/// Returns profile for Metal Shading Language numeric literals.
|
||||
///
|
||||
static Profile msl();
|
||||
|
||||
/// Constructor. Parses literal at the beginning of `_str`.
|
||||
///
|
||||
/// @param[in] _str Input string to parse. It's not copied, and it must outlive
|
||||
/// LiteralParser.
|
||||
/// @param[in] _profile Numeric syntax to accept.
|
||||
///
|
||||
LiteralParser(const StringView& _str, const Profile& _profile = defaults() );
|
||||
|
||||
/// Returns true if literal was matched.
|
||||
///
|
||||
bool isOk() const;
|
||||
|
||||
/// Returns literal that was matched, or empty string view if there was no match.
|
||||
///
|
||||
const StringView& getLiteral() const;
|
||||
|
||||
/// Returns type of literal that was matched, or `Type::Unknown` if there was no match.
|
||||
///
|
||||
Type getType() const;
|
||||
|
||||
private:
|
||||
StringView m_literal;
|
||||
Type m_type;
|
||||
bool m_ok;
|
||||
};
|
||||
|
||||
} // namespace bx
|
||||
|
||||
#include "inline/literal-parser.inl"
|
||||
|
||||
#endif // BX_LITERAL_PARSER_H_HEADER_GUARD
|
||||
@@ -29,9 +29,11 @@ namespace bx
|
||||
///
|
||||
enum class Class : uint8_t
|
||||
{
|
||||
Space, //!< Run of whitespace characters.
|
||||
Space, //!< Run of whitespace characters, line terminators included.
|
||||
SpaceHoriz, //!< Run of horizontal whitespace as `isSpaceHoriz` reads it, so up to but not including a line feed.
|
||||
NonSpace, //!< Run of non-whitespace characters.
|
||||
Identifier, //!< Run of alphanumeric characters and `_`.
|
||||
Identifier, //!< Identifier as `strIdentifier` reads one: alphanumeric characters and `_`, not starting with a digit.
|
||||
Word, //!< Word as `strWord` reads one: run of alphanumeric characters and `_`, a leading digit included.
|
||||
EndOfLine, //!< Everything up to, but not including, line terminator.
|
||||
NewLine, //!< Everything up to, and including, line terminator.
|
||||
};
|
||||
@@ -107,6 +109,19 @@ namespace bx
|
||||
///
|
||||
StringView accept(const StringView& _str);
|
||||
|
||||
/// Accept `_str` if input at cursor starts with it and it does not run into a word.
|
||||
///
|
||||
/// Unlike `accept`, which is a plain prefix test, a token ending in a word
|
||||
/// character must be followed by something that is not one, so a keyword does not
|
||||
/// match the head of a longer word: `acceptToken("if")` leaves `ifx` alone. For a
|
||||
/// token ending in punctuation the two behave the same.
|
||||
///
|
||||
/// @param[in] _str Token to match.
|
||||
///
|
||||
/// @returns Accepted string view, or empty string view if it doesn't match.
|
||||
///
|
||||
StringView acceptToken(const StringView& _str);
|
||||
|
||||
/// Accept leading run of characters belonging to `_class`.
|
||||
///
|
||||
/// @param[in] _class Character class to match.
|
||||
@@ -234,6 +249,15 @@ namespace bx
|
||||
///
|
||||
StringView getCursor(Cursor _which = Cursor::Current) const;
|
||||
|
||||
/// Returns input from cursor to end of input.
|
||||
///
|
||||
/// Where `getCursor` gives a position, this gives the range still to be scanned,
|
||||
/// which is what another parser over the same text needs handed to it.
|
||||
///
|
||||
/// @returns String view from cursor to end of input.
|
||||
///
|
||||
StringView getTail() const;
|
||||
|
||||
/// Returns text between `_from` and current cursor position.
|
||||
///
|
||||
/// @param[in] _from Position to measure from, usually obtained by `getCursor`.
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#ifndef BX_SORT_H_HEADER_GUARD
|
||||
#define BX_SORT_H_HEADER_GUARD
|
||||
|
||||
#include "allocator.h"
|
||||
#include "bx.h"
|
||||
#include "math.h"
|
||||
#include "string.h"
|
||||
@@ -74,6 +75,52 @@ namespace bx
|
||||
template<typename Ty>
|
||||
void quickSort(Ty* _data, uint32_t _num, const ComparisonFn _fn = compareAscending<Ty>);
|
||||
|
||||
/// Performs sort (Insertion Sort algorithm).
|
||||
///
|
||||
/// It's stable, it sorts in place, and it doesn't allocate. Elements move by assignment rather
|
||||
/// than by byte-wise swap, so unlike `quickSort` element type doesn't have to be trivially
|
||||
/// move assignable. It's quadratic, and it's meant for short arrays, or as a building block of
|
||||
/// `stableSort`.
|
||||
///
|
||||
/// @param _data Pointer to array data.
|
||||
/// @param _num Number of elements.
|
||||
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
|
||||
/// strict weak ordering.
|
||||
///
|
||||
template<typename Ty, typename LessFn>
|
||||
void insertionSort(Ty* _data, uint32_t _num, LessFn _less);
|
||||
|
||||
/// Performs stable sort: elements that compare equal keep their relative order.
|
||||
///
|
||||
/// Insertion sort for short runs, and bottom-up merge for the rest, ping-ponging between input
|
||||
/// and a single scratch buffer. Elements move by assignment rather than by byte-wise swap, so
|
||||
/// unlike `quickSort` element type doesn't have to be trivially move assignable. An element
|
||||
/// holding a small-string buffer, or anything else self-referential, is safe here and is not
|
||||
/// safe there.
|
||||
///
|
||||
/// @param _allocator Allocator for scratch buffer. Nothing is allocated when there are few
|
||||
/// enough elements for insertion sort alone.
|
||||
/// @param _data Pointer to array data.
|
||||
/// @param _num Number of elements.
|
||||
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
|
||||
/// strict weak ordering.
|
||||
///
|
||||
/// @attention Element type must be default constructible and copy assignable.
|
||||
///
|
||||
template<typename Ty, typename LessFn>
|
||||
void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less);
|
||||
|
||||
/// Performs stable sort over contiguous container.
|
||||
///
|
||||
/// @param _allocator Allocator for scratch buffer.
|
||||
/// @param _container Container with `empty`, `size` and `operator[]`, such as
|
||||
/// `tinystl::vector`.
|
||||
/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
|
||||
/// strict weak ordering.
|
||||
///
|
||||
template<typename ContainerT, typename LessFn>
|
||||
void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less);
|
||||
|
||||
/// Performs reordering of duplicate elements in the array in the way that unique elements
|
||||
/// are sorted to the front of array, and duplicates are after the return value index.
|
||||
///
|
||||
|
||||
@@ -420,6 +420,9 @@ namespace bx
|
||||
/// Returns string view with whitespace characters trimmed from left.
|
||||
StringView strLTrimSpace(const StringView& _str);
|
||||
|
||||
/// Returns string view with horizontal whitespace characters trimmed from left.
|
||||
StringView strLTrimSpaceHoriz(const StringView& _str);
|
||||
|
||||
/// Returns string view with non-whitespace characters trimmed from left.
|
||||
StringView strLTrimNonSpace(const StringView& _str);
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ project "bx"
|
||||
path.join(BX_DIR, "src/file.cpp"),
|
||||
path.join(BX_DIR, "src/filepath.cpp"),
|
||||
path.join(BX_DIR, "src/hash.cpp"),
|
||||
path.join(BX_DIR, "src/literal-parser.cpp"),
|
||||
path.join(BX_DIR, "src/math.cpp"),
|
||||
path.join(BX_DIR, "src/mutex.cpp"),
|
||||
path.join(BX_DIR, "src/os.cpp"),
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "file.cpp"
|
||||
#include "filepath.cpp"
|
||||
#include "hash.cpp"
|
||||
#include "literal-parser.cpp"
|
||||
#include "math.cpp"
|
||||
#include "mutex.cpp"
|
||||
#include "os.cpp"
|
||||
|
||||
381
src/literal-parser.cpp
Normal file
381
src/literal-parser.cpp
Normal file
@@ -0,0 +1,381 @@
|
||||
/*
|
||||
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
|
||||
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
#include <bx/literal-parser.h>
|
||||
#include <bx/scanner.h>
|
||||
|
||||
namespace bx
|
||||
{
|
||||
namespace
|
||||
{
|
||||
static constexpr StringView kWgslInt[] = { "i", "u", "li", "lu" };
|
||||
static constexpr StringView kWgslFloat[] = { "f", "h", "lf" };
|
||||
static constexpr StringView kGlslInt[] = { "u", "U", "lu", "LU" };
|
||||
static constexpr StringView kGlslFloat[] = { "f", "F", "lf", "LF" };
|
||||
static constexpr StringView kHlslInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "UL", "ull", "llu", "ULL", "LLU" };
|
||||
static constexpr StringView kHlslFloat[] = { "f", "F", "h", "H" };
|
||||
static constexpr StringView kCInt[] = { "u", "U", "l", "L", "ll", "LL", "ul", "lu", "uL", "Lu", "ull", "llu", "ULL", "LLU" };
|
||||
static constexpr StringView kCFloat[] = { "f", "F", "l", "L" };
|
||||
static constexpr StringView kMslFloat[] = { "f", "F", "h", "H" };
|
||||
|
||||
using LiteralProfile = LiteralParser::Profile;
|
||||
using LiteralSuffixes = LiteralParser::Suffixes;
|
||||
using LiteralType = LiteralParser::Type;
|
||||
|
||||
char peekChar(Scanner& _scanner)
|
||||
{
|
||||
const StringView view = _scanner.peek();
|
||||
|
||||
return view.isEmpty() ? '\0' : *view.getPtr();
|
||||
}
|
||||
|
||||
bool acceptDigits(Scanner& _scanner, CharTestFn _fn, char _separator)
|
||||
{
|
||||
bool ok = false;
|
||||
for (char ch = peekChar(_scanner)
|
||||
; _fn(ch) || (0 != _separator && _separator == ch)
|
||||
; ch = peekChar(_scanner) )
|
||||
{
|
||||
ok |= _fn(ch);
|
||||
_scanner.accept();
|
||||
}
|
||||
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool acceptSuffix(Scanner& _scanner, const LiteralSuffixes& _suffixes)
|
||||
{
|
||||
int32_t best = -1;
|
||||
int32_t bestLen = 0;
|
||||
for (uint16_t ii = 0; ii < _suffixes.num; ++ii)
|
||||
{
|
||||
const int32_t len = _suffixes.data[ii].getLength();
|
||||
if (len > bestLen
|
||||
&& !_scanner.peek(_suffixes.data[ii]).isEmpty() )
|
||||
{
|
||||
best = ii;
|
||||
bestLen = len;
|
||||
}
|
||||
}
|
||||
|
||||
if (0 <= best)
|
||||
{
|
||||
_scanner.accept(_suffixes.data[best]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
enum ExponentResult
|
||||
{
|
||||
ExponentNone,
|
||||
ExponentOk,
|
||||
ExponentBad,
|
||||
};
|
||||
|
||||
ExponentResult acceptExponent(Scanner& _scanner, const LiteralProfile& _profile, CharTestFn _digits)
|
||||
{
|
||||
const char ch = peekChar(_scanner);
|
||||
const bool isExponent = 'e' == ch
|
||||
|| (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch)
|
||||
;
|
||||
|
||||
if (!isExponent)
|
||||
{
|
||||
return ExponentNone;
|
||||
}
|
||||
|
||||
_scanner.accept();
|
||||
_scanner.accept('-', '+');
|
||||
|
||||
return acceptDigits(_scanner, _digits, _profile.separator) ? ExponentOk : ExponentBad;
|
||||
}
|
||||
|
||||
LiteralType parseDecimalFloat(Scanner& _scanner, const LiteralProfile& _profile)
|
||||
{
|
||||
bool dot = false;
|
||||
bool frac = false;
|
||||
if (!_scanner.accept('.').isEmpty() )
|
||||
{
|
||||
dot = true;
|
||||
frac = acceptDigits(_scanner, isNumeric, _profile.separator);
|
||||
}
|
||||
|
||||
const ExponentResult exponent = acceptExponent(_scanner, _profile, isNumeric);
|
||||
if (ExponentBad == exponent)
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
const bool ok = frac
|
||||
|| ExponentOk == exponent
|
||||
|| (dot && 0 != (_profile.flags & LiteralParser::TrailingDot) )
|
||||
;
|
||||
|
||||
if (!ok)
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.floatSuffix);
|
||||
|
||||
return LiteralType::Float;
|
||||
}
|
||||
|
||||
LiteralType parseAfterInteger(Scanner& _scanner, const LiteralProfile& _profile)
|
||||
{
|
||||
const char ch = peekChar(_scanner);
|
||||
const bool isExponent = 'e' == ch
|
||||
|| (0 != (_profile.flags & LiteralParser::UpperExponent) && 'E' == ch)
|
||||
;
|
||||
|
||||
if ('.' == ch
|
||||
|| isExponent)
|
||||
{
|
||||
return parseDecimalFloat(_scanner, _profile);
|
||||
}
|
||||
|
||||
if (0 != (_profile.flags & LiteralParser::PromoteFloatSuffix)
|
||||
&& acceptSuffix(_scanner, _profile.floatSuffix) )
|
||||
{
|
||||
return LiteralType::Float;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.intSuffix);
|
||||
|
||||
return LiteralType::Decimal;
|
||||
}
|
||||
|
||||
LiteralType parseHex(Scanner& _scanner, const LiteralProfile& _profile)
|
||||
{
|
||||
const bool anyHex = acceptDigits(_scanner, isHexNum, _profile.separator);
|
||||
|
||||
bool isFloat = false;
|
||||
bool anyHexFrac = false;
|
||||
if (0 != (_profile.flags & LiteralParser::HexFloat) )
|
||||
{
|
||||
if (!_scanner.accept('.').isEmpty() )
|
||||
{
|
||||
isFloat = true;
|
||||
anyHexFrac = acceptDigits(_scanner, isHexNum, _profile.separator);
|
||||
}
|
||||
|
||||
const char ch = peekChar(_scanner);
|
||||
if ('p' == ch
|
||||
|| 'P' == ch)
|
||||
{
|
||||
isFloat = true;
|
||||
_scanner.accept();
|
||||
_scanner.accept('-', '+');
|
||||
|
||||
if (!acceptDigits(_scanner, isNumeric, _profile.separator) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isFloat)
|
||||
{
|
||||
// `0x.p2` has no significand digits.
|
||||
if (!anyHex
|
||||
&& !anyHexFrac)
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.floatSuffix);
|
||||
|
||||
return LiteralType::Float;
|
||||
}
|
||||
|
||||
// `0x` with no digits.
|
||||
if (!anyHex)
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.intSuffix);
|
||||
|
||||
return LiteralType::Hexadecimal;
|
||||
}
|
||||
|
||||
LiteralType parseLiteral(Scanner& _scanner, const LiteralProfile& _profile)
|
||||
{
|
||||
if (0 != (_profile.flags & LiteralParser::Sign) )
|
||||
{
|
||||
_scanner.accept('-', '+');
|
||||
}
|
||||
|
||||
if (!_scanner.accept('0').isEmpty() )
|
||||
{
|
||||
if (0 != (_profile.flags & LiteralParser::BinPrefix)
|
||||
&& !_scanner.accept('b').isEmpty() )
|
||||
{
|
||||
if (!acceptDigits(_scanner, isBinNum, _profile.separator) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.intSuffix);
|
||||
|
||||
return LiteralType::Binary;
|
||||
}
|
||||
|
||||
if (0 != (_profile.flags & LiteralParser::OctPrefix)
|
||||
&& !_scanner.accept('o').isEmpty() )
|
||||
{
|
||||
if (!acceptDigits(_scanner, isOctNum, _profile.separator) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.intSuffix);
|
||||
|
||||
return LiteralType::Octal;
|
||||
}
|
||||
|
||||
if (0 != (_profile.flags & LiteralParser::HexPrefix)
|
||||
&& !_scanner.accept('x', 'X').isEmpty() )
|
||||
{
|
||||
return parseHex(_scanner, _profile);
|
||||
}
|
||||
|
||||
const bool leadingZeros = acceptDigits(_scanner, isNumeric, _profile.separator);
|
||||
const LiteralType type = parseAfterInteger(_scanner, _profile);
|
||||
|
||||
if (leadingZeros
|
||||
&& LiteralType::Decimal == type
|
||||
&& 0 != (_profile.flags & LiteralParser::NoLeadingZero) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
if (isNumeric(peekChar(_scanner) ) )
|
||||
{
|
||||
acceptDigits(_scanner, isNumeric, _profile.separator);
|
||||
|
||||
return parseAfterInteger(_scanner, _profile);
|
||||
}
|
||||
|
||||
if (0 != (_profile.flags & LiteralParser::LeadingDot)
|
||||
&& '.' == peekChar(_scanner) )
|
||||
{
|
||||
_scanner.accept('.');
|
||||
|
||||
// `.` must be followed by a digit.
|
||||
if (!acceptDigits(_scanner, isNumeric, _profile.separator) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
if (ExponentBad == acceptExponent(_scanner, _profile, isNumeric) )
|
||||
{
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
acceptSuffix(_scanner, _profile.floatSuffix);
|
||||
|
||||
return LiteralType::Float;
|
||||
}
|
||||
|
||||
return LiteralType::Unknown;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
LiteralParser::Profile LiteralParser::defaults()
|
||||
{
|
||||
return
|
||||
{
|
||||
Sign | BinPrefix | OctPrefix | HexPrefix,
|
||||
'_',
|
||||
{ NULL, 0 },
|
||||
{ NULL, 0 },
|
||||
};
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::wgsl()
|
||||
{
|
||||
return
|
||||
{
|
||||
HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix | NoLeadingZero,
|
||||
0,
|
||||
{ kWgslInt, uint16_t(BX_COUNTOF(kWgslInt) ) },
|
||||
{ kWgslFloat, uint16_t(BX_COUNTOF(kWgslFloat) ) },
|
||||
};
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::glsl()
|
||||
{
|
||||
return
|
||||
{
|
||||
HexPrefix | LeadingDot | TrailingDot | UpperExponent,
|
||||
0,
|
||||
{ kGlslInt, uint16_t(BX_COUNTOF(kGlslInt) ) },
|
||||
{ kGlslFloat, uint16_t(BX_COUNTOF(kGlslFloat) ) },
|
||||
};
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::hlsl()
|
||||
{
|
||||
return
|
||||
{
|
||||
HexPrefix | LeadingDot | TrailingDot | UpperExponent | PromoteFloatSuffix,
|
||||
0,
|
||||
{ kHlslInt, uint16_t(BX_COUNTOF(kHlslInt) ) },
|
||||
{ kHlslFloat, uint16_t(BX_COUNTOF(kHlslFloat) ) },
|
||||
};
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::c()
|
||||
{
|
||||
return
|
||||
{
|
||||
HexPrefix | HexFloat | LeadingDot | TrailingDot | UpperExponent,
|
||||
0,
|
||||
{ kCInt, uint16_t(BX_COUNTOF(kCInt) ) },
|
||||
{ kCFloat, uint16_t(BX_COUNTOF(kCFloat) ) },
|
||||
};
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::cpp()
|
||||
{
|
||||
Profile profile = c();
|
||||
profile.flags |= BinPrefix;
|
||||
profile.separator = '\'';
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
LiteralParser::Profile LiteralParser::msl()
|
||||
{
|
||||
Profile profile = cpp();
|
||||
profile.floatSuffix = { kMslFloat, uint16_t(BX_COUNTOF(kMslFloat) ) };
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
LiteralParser::LiteralParser(const StringView& _str, const Profile& _profile)
|
||||
: m_type(Type::Unknown)
|
||||
, m_ok(false)
|
||||
{
|
||||
Scanner scanner(_str);
|
||||
const StringView start = scanner.getCursor();
|
||||
|
||||
m_type = parseLiteral(scanner, _profile);
|
||||
m_ok = Type::Unknown != m_type;
|
||||
m_literal = m_ok
|
||||
? scanner.between(start)
|
||||
: StringView()
|
||||
;
|
||||
}
|
||||
|
||||
} // namespace bx
|
||||
@@ -90,6 +90,9 @@ namespace bx
|
||||
case Class::NonSpace:
|
||||
return strLTrimNonSpace(m_tail);
|
||||
|
||||
case Class::SpaceHoriz:
|
||||
return strLTrimSpaceHoriz(m_tail);
|
||||
|
||||
case Class::Identifier:
|
||||
{
|
||||
const StringView word = strIdentifier(m_tail);
|
||||
@@ -99,6 +102,15 @@ namespace bx
|
||||
;
|
||||
}
|
||||
|
||||
case Class::Word:
|
||||
{
|
||||
const StringView word = strWord(m_tail);
|
||||
return word.isEmpty()
|
||||
? m_tail
|
||||
: StringView(word.getTerm(), m_tail.getTerm() )
|
||||
;
|
||||
}
|
||||
|
||||
case Class::EndOfLine:
|
||||
return strFindEol(m_tail);
|
||||
|
||||
|
||||
@@ -410,6 +410,19 @@ namespace bx
|
||||
return StringView(_str.getTerm(), _str.getTerm() );
|
||||
}
|
||||
|
||||
StringView strLTrimSpaceHoriz(const StringView& _str)
|
||||
{
|
||||
for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr)
|
||||
{
|
||||
if (!isSpaceHoriz(*ptr) )
|
||||
{
|
||||
return StringView(ptr, term);
|
||||
}
|
||||
}
|
||||
|
||||
return StringView(_str.getTerm(), _str.getTerm() );
|
||||
}
|
||||
|
||||
StringView strLTrimNonSpace(const StringView& _str)
|
||||
{
|
||||
for (const char* ptr = _str.getPtr(), *term = _str.getTerm(); ptr != term; ++ptr)
|
||||
|
||||
236
tests/literal-parser_test.cpp
Normal file
236
tests/literal-parser_test.cpp
Normal file
@@ -0,0 +1,236 @@
|
||||
/*
|
||||
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
|
||||
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
#include "test.h"
|
||||
|
||||
#include <bx/literal-parser.h>
|
||||
|
||||
bool isNumber(bx::StringView _str)
|
||||
{
|
||||
const char* str = _str.getPtr();
|
||||
int32_t len = _str.getLength();
|
||||
|
||||
const char* end = str + len;
|
||||
const char* p = str;
|
||||
|
||||
if (p < end && (*p == '-' || *p == '+'))
|
||||
{
|
||||
p++;
|
||||
}
|
||||
|
||||
bool has_digit = false;
|
||||
|
||||
while (p < end && isdigit((unsigned char)*p))
|
||||
{
|
||||
has_digit = true;
|
||||
p++;
|
||||
}
|
||||
|
||||
if (!has_digit) return false;
|
||||
|
||||
if (p < end && *p == '.')
|
||||
{
|
||||
p++;
|
||||
has_digit = false;
|
||||
|
||||
while (p < end && isdigit((unsigned char)*p))
|
||||
{
|
||||
has_digit = true;
|
||||
p++;
|
||||
}
|
||||
|
||||
if (!has_digit) return false;
|
||||
}
|
||||
|
||||
if (p < end && (*p == 'e' || *p == 'E'))
|
||||
{
|
||||
p++;
|
||||
|
||||
if (p < end && (*p == '-' || *p == '+'))
|
||||
{
|
||||
p++;
|
||||
}
|
||||
|
||||
has_digit = false;
|
||||
|
||||
while (p < end && isdigit((unsigned char)*p))
|
||||
{
|
||||
has_digit = true;
|
||||
p++;
|
||||
}
|
||||
|
||||
if (!has_digit) return false;
|
||||
}
|
||||
|
||||
return p == end;
|
||||
}
|
||||
|
||||
bool testNumber(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown)
|
||||
{
|
||||
bx::LiteralParser p(_input);
|
||||
|
||||
const bx::StringView literal = p.getLiteral();
|
||||
|
||||
bool ok = true
|
||||
&& p.isOk()
|
||||
&& _type == p.getType()
|
||||
&& bx::isEqual(literal, _output)
|
||||
;
|
||||
|
||||
if (!ok
|
||||
&& _type != bx::LiteralParser::Type::Unknown)
|
||||
{
|
||||
DBG("%d: %S, expected %d: %S", p.getType(), &literal, _type, &_output);
|
||||
}
|
||||
|
||||
BX_TRACE("is_number %d, %d <- %S"
|
||||
, isNumber(literal)
|
||||
, ok
|
||||
, &literal
|
||||
);
|
||||
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool testNumber(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type = bx::LiteralParser::Type::Unknown)
|
||||
{
|
||||
return testNumber(_inputAndOutput, _inputAndOutput, _type);
|
||||
}
|
||||
|
||||
TEST_CASE("LiteralParser", "[parser]")
|
||||
{
|
||||
// error
|
||||
REQUIRE(!testNumber("") );
|
||||
REQUIRE(!testNumber(".") );
|
||||
REQUIRE(!testNumber("abvgd") );
|
||||
REQUIRE(!testNumber("-e0") );
|
||||
REQUIRE(!testNumber("-e-89") );
|
||||
REQUIRE(!testNumber("0b") );
|
||||
REQUIRE(!testNumber("0o") );
|
||||
REQUIRE(!testNumber("0x") );
|
||||
REQUIRE(!testNumber("0e") );
|
||||
REQUIRE(!testNumber("-0.e-") );
|
||||
REQUIRE(!testNumber("1.38913e-") );
|
||||
|
||||
// decimal
|
||||
REQUIRE(testNumber("0", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("00", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("000", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0000", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("00_00", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0_0_0_0", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0-", "0", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0-e0", "0", bx::LiteralParser::Type::Decimal) );
|
||||
|
||||
// binary
|
||||
REQUIRE(testNumber("0b0001001110001001", bx::LiteralParser::Type::Binary) );
|
||||
REQUIRE(testNumber("0b0001_0011_1000_1001", bx::LiteralParser::Type::Binary) );
|
||||
REQUIRE(testNumber("0b0;", "0b0", bx::LiteralParser::Type::Binary) );
|
||||
REQUIRE(testNumber("0b0+", "0b0", bx::LiteralParser::Type::Binary) );
|
||||
|
||||
// octal
|
||||
REQUIRE(testNumber("02555", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0o2555", bx::LiteralParser::Type::Octal) );
|
||||
REQUIRE(testNumber("0o25_55", bx::LiteralParser::Type::Octal) );
|
||||
|
||||
// hex
|
||||
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0x1389", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0x0;", "0x0", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0x0+", "0x0", bx::LiteralParser::Type::Hexadecimal) );
|
||||
|
||||
// integer
|
||||
REQUIRE(testNumber("1389", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("13_89", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0;", "0", bx::LiteralParser::Type::Decimal) );
|
||||
REQUIRE(testNumber("0+", "0", bx::LiteralParser::Type::Decimal) );
|
||||
|
||||
// floating point
|
||||
REQUIRE(testNumber("0.1389", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("-0.1389", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("+0.1389", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("0.e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("-0.e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("+0.e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("138e-9", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("1.38913e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("1.389e+001", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("1.389e-002", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("1.389e-002;", "1.389e-002", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("-0.e-89;", "-0.e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("-0.e-89+", "-0.e-89", bx::LiteralParser::Type::Float) );
|
||||
REQUIRE(testNumber("-0.e-89e", "-0.e-89", bx::LiteralParser::Type::Float) );
|
||||
|
||||
// hex with letters (lower/upper/mixed) and digit separators
|
||||
REQUIRE(testNumber("0xabcdef", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xABCDEF", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xDeadC0de", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xff", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0x13_89", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xFF_FF", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xdead;", "0xdead", bx::LiteralParser::Type::Hexadecimal) );
|
||||
REQUIRE(testNumber("0xC0DE-", "0xC0DE", bx::LiteralParser::Type::Hexadecimal) );
|
||||
|
||||
// more errors: a lone sign or a leading separator is not a number
|
||||
REQUIRE(!testNumber("+") );
|
||||
REQUIRE(!testNumber("-") );
|
||||
REQUIRE(!testNumber("_5") );
|
||||
REQUIRE(!testNumber("0xg") );
|
||||
}
|
||||
|
||||
static bool testWgsl(const bx::StringView& _input, const bx::StringView& _output, bx::LiteralParser::Type _type)
|
||||
{
|
||||
bx::LiteralParser p(_input, bx::LiteralParser::wgsl() );
|
||||
|
||||
return p.isOk()
|
||||
&& _type == p.getType()
|
||||
&& bx::isEqual(p.getLiteral(), _output)
|
||||
;
|
||||
}
|
||||
|
||||
static bool testWgsl(const bx::StringView& _inputAndOutput, bx::LiteralParser::Type _type)
|
||||
{
|
||||
return testWgsl(_inputAndOutput, _inputAndOutput, _type);
|
||||
}
|
||||
|
||||
TEST_CASE("LiteralParser WGSL profile", "[parser]")
|
||||
{
|
||||
typedef bx::LiteralParser LP;
|
||||
|
||||
// integers (+ WGSL i/u/li/lu suffixes) and hex
|
||||
REQUIRE(testWgsl("0", LP::Decimal) );
|
||||
REQUIRE(testWgsl("42", LP::Decimal) );
|
||||
REQUIRE(testWgsl("1u", LP::Decimal) );
|
||||
REQUIRE(testWgsl("1i", LP::Decimal) );
|
||||
REQUIRE(testWgsl("1li", LP::Decimal) );
|
||||
REQUIRE(testWgsl("1lu", LP::Decimal) );
|
||||
REQUIRE(testWgsl("0xFF", LP::Hexadecimal) );
|
||||
REQUIRE(testWgsl("0xffu", LP::Hexadecimal) );
|
||||
|
||||
// floats: fraction/exponent, edge dots, suffixes, hex float
|
||||
REQUIRE(testWgsl("1.0", LP::Float) );
|
||||
REQUIRE(testWgsl(".5", LP::Float) );
|
||||
REQUIRE(testWgsl("2.", LP::Float) );
|
||||
REQUIRE(testWgsl("1e5", LP::Float) );
|
||||
REQUIRE(testWgsl("3.14e-2", LP::Float) );
|
||||
REQUIRE(testWgsl("1.5f", LP::Float) );
|
||||
REQUIRE(testWgsl("1f", LP::Float) ); // bare integer + float suffix -> float
|
||||
REQUIRE(testWgsl("1h", LP::Float) );
|
||||
REQUIRE(testWgsl("1lf", LP::Float) );
|
||||
REQUIRE(testWgsl("0x1p-3", LP::Float) );
|
||||
REQUIRE(testWgsl("0x1.8p3", LP::Float) );
|
||||
|
||||
// prefix match: trailing input is not part of the literal
|
||||
REQUIRE(testWgsl("42;", "42", LP::Decimal) );
|
||||
REQUIRE(testWgsl("0xFFu-", "0xFFu", LP::Hexadecimal) );
|
||||
REQUIRE(testWgsl("1.x", "1.", LP::Float) ); // "1." is a float (trailing dot), then ".x"
|
||||
|
||||
// WGSL has no sign, no 0b/0o prefixes, no '_' separators
|
||||
REQUIRE(!bx::LiteralParser("-5", LP::wgsl() ).isOk() ); // sign is a separate token
|
||||
REQUIRE(testWgsl("0b1", "0", LP::Decimal) ); // "0" then "b1"
|
||||
REQUIRE(testWgsl("5_5", "5", LP::Decimal) ); // "5" then "_5"
|
||||
}
|
||||
@@ -390,3 +390,112 @@ TEST_CASE("LineReader", "[scanner][string]")
|
||||
REQUIRE(second.getPtr() == input.getPtr() + 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Scanner.class Word", "[scanner]")
|
||||
{
|
||||
{
|
||||
bx::Scanner sc("2d, f32");
|
||||
|
||||
REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) );
|
||||
REQUIRE("," == sc.accept(',') );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("2d");
|
||||
|
||||
REQUIRE(sc.accept(bx::Scanner::Class::Identifier).isEmpty() );
|
||||
REQUIRE("2d" == sc.accept(bx::Scanner::Class::Word) );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("_leading and9digits");
|
||||
|
||||
REQUIRE("_leading" == sc.accept(bx::Scanner::Class::Word) );
|
||||
REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) );
|
||||
REQUIRE("and9digits" == sc.accept(bx::Scanner::Class::Word) );
|
||||
REQUIRE(sc.isDone() );
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Scanner.class SpaceHoriz", "[scanner]")
|
||||
{
|
||||
{
|
||||
bx::Scanner horiz(" \t\nx");
|
||||
|
||||
REQUIRE(" \t" == horiz.accept(bx::Scanner::Class::SpaceHoriz) );
|
||||
REQUIRE("\n" == horiz.accept('\n') );
|
||||
|
||||
bx::Scanner space(" \t\nx");
|
||||
|
||||
REQUIRE(" \t\n" == space.accept(bx::Scanner::Class::Space) );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("x");
|
||||
|
||||
REQUIRE(sc.accept(bx::Scanner::Class::SpaceHoriz).isEmpty() );
|
||||
REQUIRE("x" == sc.accept('x') );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("a \r\nb");
|
||||
|
||||
REQUIRE("a" == sc.accept(bx::Scanner::Class::Word) );
|
||||
REQUIRE(" \r" == sc.accept(bx::Scanner::Class::SpaceHoriz) );
|
||||
REQUIRE("\n" == sc.accept('\n') );
|
||||
REQUIRE("b" == sc.accept(bx::Scanner::Class::Word) );
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Scanner.acceptToken", "[scanner]")
|
||||
{
|
||||
{
|
||||
bx::Scanner bare("ifx");
|
||||
REQUIRE("if" == bare.accept(bx::StringView("if") ) );
|
||||
|
||||
bx::Scanner token("ifx");
|
||||
REQUIRE(token.acceptToken(bx::StringView("if") ).isEmpty() );
|
||||
REQUIRE("ifx" == token.accept(bx::Scanner::Class::Word) );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("if (x)");
|
||||
|
||||
REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) );
|
||||
REQUIRE(" " == sc.accept(bx::Scanner::Class::SpaceHoriz) );
|
||||
REQUIRE("(" == sc.acceptToken(bx::StringView("(") ) );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("->x");
|
||||
|
||||
REQUIRE("->" == sc.acceptToken(bx::StringView("->") ) );
|
||||
REQUIRE("x" == sc.accept(bx::Scanner::Class::Word) );
|
||||
}
|
||||
|
||||
{
|
||||
bx::Scanner sc("if");
|
||||
|
||||
REQUIRE("if" == sc.acceptToken(bx::StringView("if") ) );
|
||||
REQUIRE(sc.isDone() );
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Scanner.getTail", "[scanner]")
|
||||
{
|
||||
const bx::StringView input("abc def");
|
||||
|
||||
bx::Scanner sc(input);
|
||||
|
||||
REQUIRE(input == sc.getTail() );
|
||||
|
||||
sc.accept(bx::Scanner::Class::Word);
|
||||
|
||||
REQUIRE(" def" == sc.getTail() );
|
||||
REQUIRE(sc.getTail().getPtr() == input.getPtr() + 3);
|
||||
REQUIRE(sc.getTail().getTerm() == input.getTerm() );
|
||||
|
||||
sc.acceptAll();
|
||||
|
||||
REQUIRE(sc.getTail().isEmpty() );
|
||||
}
|
||||
|
||||
@@ -181,3 +181,199 @@ TEST_CASE("ComparisonFn", "[sort]")
|
||||
compareTest< float>(-13.89f, 1389.0f);
|
||||
compareTest<double>(-13.89f, 1389.0f);
|
||||
}
|
||||
|
||||
#include <bx/allocator.h>
|
||||
#include <tinystl/allocator.h>
|
||||
#include <tinystl/string.h>
|
||||
#include <tinystl/vector.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
struct Item
|
||||
{
|
||||
uint32_t key;
|
||||
uint32_t seq;
|
||||
};
|
||||
|
||||
bool byKey(const Item& _a, const Item& _b)
|
||||
{
|
||||
return _a.key < _b.key;
|
||||
}
|
||||
|
||||
tinystl::vector<Item> makeItems(uint32_t _num, uint32_t _numKeys)
|
||||
{
|
||||
tinystl::vector<Item> items;
|
||||
uint32_t rng = 12345;
|
||||
for (uint32_t ii = 0; ii < _num; ++ii)
|
||||
{
|
||||
rng = rng*1664525 + 1013904223;
|
||||
const Item item =
|
||||
{
|
||||
.key = (rng >> 16) % _numKeys,
|
||||
.seq = ii,
|
||||
};
|
||||
|
||||
items.push_back(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
void checkSortedAndStable(tinystl::vector<Item>& _items)
|
||||
{
|
||||
bx::DefaultAllocator allocator;
|
||||
|
||||
const uint32_t num = uint32_t(_items.size() );
|
||||
bx::stableSort(&allocator, _items, byKey);
|
||||
|
||||
REQUIRE(num == _items.size() );
|
||||
|
||||
for (uint32_t ii = 1; ii < num; ++ii)
|
||||
{
|
||||
REQUIRE(_items[ii-1].key <= _items[ii].key);
|
||||
|
||||
if (_items[ii-1].key == _items[ii].key)
|
||||
{
|
||||
REQUIRE(_items[ii-1].seq < _items[ii].seq);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("insertionSort", "[sort]")
|
||||
{
|
||||
Item items[] =
|
||||
{
|
||||
{ 3, 0 }, { 1, 1 }, { 3, 2 }, { 0, 3 }, { 1, 4 }, { 3, 5 },
|
||||
};
|
||||
|
||||
bx::insertionSort(items, BX_COUNTOF(items), byKey);
|
||||
|
||||
const uint32_t expectedKey[] = { 0, 1, 1, 3, 3, 3 };
|
||||
const uint32_t expectedSeq[] = { 3, 1, 4, 0, 2, 5 };
|
||||
for (uint32_t ii = 0; ii < BX_COUNTOF(items); ++ii)
|
||||
{
|
||||
REQUIRE(expectedKey[ii] == items[ii].key);
|
||||
REQUIRE(expectedSeq[ii] == items[ii].seq);
|
||||
}
|
||||
|
||||
bx::insertionSort(items, 0, byKey);
|
||||
bx::insertionSort(items, 1, byKey);
|
||||
REQUIRE(0 == items[0].key);
|
||||
}
|
||||
|
||||
TEST_CASE("stableSort is stable across the insertion/merge threshold", "[sort]")
|
||||
{
|
||||
const uint32_t sizes[] = { 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000 };
|
||||
|
||||
for (uint32_t ii = 0; ii < BX_COUNTOF(sizes); ++ii)
|
||||
{
|
||||
tinystl::vector<Item> items = makeItems(sizes[ii], 4);
|
||||
checkSortedAndStable(items);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("stableSort handles degenerate orderings", "[sort]")
|
||||
{
|
||||
bx::DefaultAllocator allocator;
|
||||
|
||||
{
|
||||
tinystl::vector<Item> items;
|
||||
for (uint32_t ii = 0; ii < 100; ++ii)
|
||||
{
|
||||
const Item item = { ii, ii };
|
||||
items.push_back(item);
|
||||
}
|
||||
|
||||
checkSortedAndStable(items);
|
||||
REQUIRE(0 == items[0].key);
|
||||
REQUIRE(99 == items[99].key);
|
||||
}
|
||||
|
||||
{
|
||||
tinystl::vector<Item> items;
|
||||
for (uint32_t ii = 0; ii < 100; ++ii)
|
||||
{
|
||||
const Item item = { 100-ii, ii };
|
||||
items.push_back(item);
|
||||
}
|
||||
|
||||
bx::stableSort(&allocator, items, byKey);
|
||||
for (uint32_t ii = 1; ii < 100; ++ii)
|
||||
{
|
||||
REQUIRE(items[ii-1].key < items[ii].key);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
tinystl::vector<Item> items;
|
||||
for (uint32_t ii = 0; ii < 100; ++ii)
|
||||
{
|
||||
const Item item = { 7, ii };
|
||||
items.push_back(item);
|
||||
}
|
||||
|
||||
bx::stableSort(&allocator, items, byKey);
|
||||
for (uint32_t ii = 0; ii < 100; ++ii)
|
||||
{
|
||||
REQUIRE(ii == items[ii].seq);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("stableSort moves elements by assignment", "[sort]")
|
||||
{
|
||||
struct Named
|
||||
{
|
||||
uint32_t key;
|
||||
tinystl::string name;
|
||||
};
|
||||
|
||||
const tinystl::string prefix("a-long-name-that-will-not-fit-in-a-small-string-buffer-");
|
||||
|
||||
bx::DefaultAllocator allocator;
|
||||
|
||||
tinystl::vector<Named> items;
|
||||
for (uint32_t ii = 0; ii < 200; ++ii)
|
||||
{
|
||||
const char suffix[] = { char('0' + ii%10) };
|
||||
|
||||
Named item;
|
||||
item.key = (200 - ii) % 10;
|
||||
item.name = prefix;
|
||||
item.name.append(suffix, suffix + 1);
|
||||
|
||||
items.push_back(item);
|
||||
}
|
||||
|
||||
bx::stableSort(&allocator, items, [](const Named& _a, const Named& _b) { return _a.key < _b.key; });
|
||||
|
||||
REQUIRE(200 == items.size() );
|
||||
for (uint32_t ii = 1; ii < items.size(); ++ii)
|
||||
{
|
||||
REQUIRE(items[ii-1].key <= items[ii].key);
|
||||
}
|
||||
|
||||
for (uint32_t ii = 0; ii < items.size(); ++ii)
|
||||
{
|
||||
REQUIRE(prefix.size() + 1 == items[ii].name.size() );
|
||||
REQUIRE(0 == bx::memCmp(items[ii].name.c_str(), prefix.c_str(), prefix.size() ) );
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("stableSort handles empty and single-element containers", "[sort]")
|
||||
{
|
||||
bx::DefaultAllocator allocator;
|
||||
|
||||
tinystl::vector<Item> empty;
|
||||
bx::stableSort(&allocator, empty, byKey);
|
||||
REQUIRE(empty.empty() );
|
||||
|
||||
tinystl::vector<Item> one;
|
||||
const Item item = { 42, 0 };
|
||||
one.push_back(item);
|
||||
bx::stableSort(&allocator, one, byKey);
|
||||
REQUIRE(1 == one.size() );
|
||||
REQUIRE(42 == one[0].key);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user