Added Class::Word, Class::SpaceHoriz, acceptToken and getTail to Scanner. Added LiteralParser. Added stableSort and insertionSort. (#432)

include/bx/inline/literal-parser.inl

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2011-2026 Branimir Karadzic. All rights reserved.
+ * License: https://github.com/bkaradzic/bx/blob/master/LICENSE
+ */
+
+#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
+#	error "Must be included from bx/literal-parser.h!"
+#endif // BX_LITERAL_PARSER_H_HEADER_GUARD
+
+namespace bx
+{
+	inline bool LiteralParser::isOk() const
+	{
+		return m_ok;
+	}
+
+	inline const StringView& LiteralParser::getLiteral() const
+	{
+		return m_literal;
+	}
+
+	inline LiteralParser::Type LiteralParser::getType() const
+	{
+		return m_type;
+	}
+
+} // namespace bx

include/bx/inline/scanner.inl

@@ -85,6 +85,31 @@ namespace bx
 		return getCursor();
 	}
 
+	inline StringView Scanner::acceptToken(const StringView& _str)
+	{
+		if (!hasPrefix(m_tail, _str) )
+		{
+			return getCursor();
+		}
+
+		const char   last = _str.getTerm()[-1];
+		const char* after = m_tail.getPtr() + _str.getLength();
+
+		if ( (isAlphaNum(last) || '_' == last)
+		&&  after < m_input.getTerm()
+		&& (isAlphaNum(*after) || '_' == *after) )
+		{
+			return getCursor();
+		}
+
+		return moveTo({ m_tail.getPtr(), _str.getLength() });
+	}
+
+	inline StringView Scanner::getTail() const
+	{
+		return m_tail;
+	}
+
 	inline StringView Scanner::accept(Class _class)
 	{
 		return moveTo({ m_tail.getPtr(), strFunc(_class).getPtr() });

include/bx/inline/sort.inl

@@ -55,6 +55,119 @@ namespace bx
 		quickSort( (void*)_data, _num, sizeof(Ty), _fn);
 	}
 
+	constexpr uint32_t kStableSortInsertionRun = 32;
+
+	template<typename Ty, typename LessFn>
+	inline void insertionSort(Ty* _data, uint32_t _num, LessFn _less)
+	{
+		for (uint32_t ii = 1; ii < _num; ++ii)
+		{
+			Ty item = _data[ii];
+
+			uint32_t jj = ii;
+			for (; jj > 0 && _less(item, _data[jj-1]); --jj)
+			{
+				_data[jj] = _data[jj-1];
+			}
+
+			_data[jj] = item;
+		}
+	}
+
+	template<typename Ty, typename LessFn>
+	inline void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less)
+	{
+		if (2 > _num)
+		{
+			return;
+		}
+
+		for (uint32_t ii = 0; ii < _num; ii += kStableSortInsertionRun)
+		{
+			insertionSort(_data + ii, min(kStableSortInsertionRun, _num - ii), _less);
+		}
+
+		if (kStableSortInsertionRun >= _num)
+		{
+			return;
+		}
+
+		Ty* scratch = (Ty*)alloc(_allocator, _num*sizeof(Ty) );
+		for (uint32_t ii = 0; ii < _num; ++ii)
+		{
+			BX_PLACEMENT_NEW(&scratch[ii], Ty);
+		}
+
+		Ty* src = _data;
+		Ty* dst = scratch;
+
+		for (uint32_t width = kStableSortInsertionRun; width < _num; width <<= 1)
+		{
+			for (uint32_t ii = 0; ii < _num; ii += width<<1)
+			{
+				const uint32_t mid = min(ii +  width,     _num);
+				const uint32_t end = min(ii + (width<<1), _num);
+
+				uint32_t ll = ii;
+				uint32_t rr = mid;
+				uint32_t oo = ii;
+
+				while (ll < mid
+				&&     rr < end)
+				{
+					if (_less(src[rr], src[ll]) )
+					{
+						dst[oo] = src[rr];
+						++rr;
+					}
+					else
+					{
+						dst[oo] = src[ll];
+						++ll;
+					}
+
+					++oo;
+				}
+
+				for (; ll < mid; ++ll, ++oo)
+				{
+					dst[oo] = src[ll];
+				}
+
+				for (; rr < end; ++rr, ++oo)
+				{
+					dst[oo] = src[rr];
+				}
+			}
+
+			swap(src, dst);
+		}
+
+		if (src != _data)
+		{
+			for (uint32_t ii = 0; ii < _num; ++ii)
+			{
+				_data[ii] = src[ii];
+			}
+		}
+
+		for (uint32_t ii = 0; ii < _num; ++ii)
+		{
+			scratch[ii].~Ty();
+		}
+
+		free(_allocator, scratch);
+	}
+
+	template<typename ContainerT, typename LessFn>
+	inline void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less)
+	{
+		if (!_container.empty() )
+		{
+			stableSort(_allocator, &_container[0], uint32_t(_container.size() ), _less);
+		}
+	}
+
 	template<typename Ty>
 	inline uint32_t unique(void* _data, uint32_t _num, uint32_t _stride, const ComparisonFn _fn)
 	{

include/bx/literal-parser.h

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2011-2026 Branimir Karadzic. All rights reserved.
+ * License: https://github.com/bkaradzic/bx/blob/master/LICENSE
+ */
+
+#ifndef BX_LITERAL_PARSER_H_HEADER_GUARD
+#define BX_LITERAL_PARSER_H_HEADER_GUARD
+
+#include <bx/string.h>
+
+namespace bx
+{
+	/// Parses numeric literal at the beginning of a string.
+	///
+	/// Syntax that's accepted is selected by `Profile`, so that numeric literals of different
+	/// languages can be read by the same parser. Profiles for a few languages are provided by
+	/// `wgsl`, `glsl`, `hlsl`, `c`, `cpp`, and `msl`.
+	///
+	/// Parsing stops at the first character that can't be part of literal, and trailing input is
+	/// not an error. Literal that was matched is returned by `getLiteral`, and it's a view into
+	/// input string.
+	///
+	/// @attention LiteralParser doesn't own input string. Input string must outlive LiteralParser.
+	///
+	class LiteralParser
+	{
+		BX_CLASS(LiteralParser
+			, NO_DEFAULT_CTOR
+			, NO_COPY
+			);
+
+	public:
+		/// Type of literal that was matched.
+		///
+		enum Type : uint8_t
+		{
+			Unknown,     //!< Input doesn't start with literal that profile accepts.
+
+			Binary,      //!< Binary integer, `0b1010`.
+			Octal,       //!< Octal integer, `0o777`.
+			Decimal,     //!< Decimal integer, `42`.
+			Hexadecimal, //!< Hexadecimal integer, `0x2a`.
+			Float,       //!< Floating point number, `1.5`, `1e3`, or `0x1.8p3`.
+		};
+
+		/// Syntax flags, see `Profile::flags`.
+		///
+		enum : uint32_t
+		{
+			Sign               = 1u<<0, //!< Leading `+` or `-` is part of literal.
+			HexPrefix          = 1u<<1, //!< Hexadecimal `0x` and `0X` prefix.
+			BinPrefix          = 1u<<2, //!< Binary `0b` prefix.
+			OctPrefix          = 1u<<3, //!< Octal `0o` prefix.
+			HexFloat           = 1u<<4, //!< Hexadecimal float, `0x1.8p3`. It requires `.` or `p`.
+			LeadingDot         = 1u<<5, //!< Float without integer part, `.5`.
+			TrailingDot        = 1u<<6, //!< Float without fractional part, `2.`.
+			UpperExponent      = 1u<<7, //!< `E` and `P` exponent, in addition to `e` and `p`.
+			PromoteFloatSuffix = 1u<<8, //!< Float suffix makes decimal integer a float, WGSL `1f`.
+			NoLeadingZero      = 1u<<9, //!< Decimal integer may not have leading zero. WGSL rejects `0123`, but `00012.` is a valid float.
+		};
+
+		/// Set of suffixes that literal can end with.
+		///
+		struct Suffixes
+		{
+			const StringView* data; //!< Suffixes to match. Longest match wins.
+			uint16_t          num;  //!< Number of suffixes.
+		};
+
+		/// Numeric syntax that LiteralParser accepts.
+		///
+		struct Profile
+		{
+			uint32_t flags;       //!< Syntax flags.
+			char     separator;   //!< Digit separator, or 0 if there is none. For example `_` or `'`.
+			Suffixes intSuffix;   //!< Suffixes accepted on integer literal.
+			Suffixes floatSuffix; //!< Suffixes accepted on float literal.
+		};
+
+		/// Returns profile that accepts sign, binary, octal and hexadecimal prefix, and `_` digit
+		/// separator, with no suffixes.
+		///
+		static Profile defaults();
+
+		/// Returns profile for WGSL numeric literals.
+		///
+		static Profile wgsl();
+
+		/// Returns profile for GLSL numeric literals.
+		///
+		static Profile glsl();
+
+		/// Returns profile for HLSL numeric literals.
+		///
+		static Profile hlsl();
+
+		/// Returns profile for C numeric literals.
+		///
+		static Profile c();
+
+		/// Returns profile for C++ numeric literals.
+		///
+		static Profile cpp();
+
+		/// Returns profile for Metal Shading Language numeric literals.
+		///
+		static Profile msl();
+
+		/// Constructor. Parses literal at the beginning of `_str`.
+		///
+		/// @param[in] _str Input string to parse. It's not copied, and it must outlive
+		///   LiteralParser.
+		/// @param[in] _profile Numeric syntax to accept.
+		///
+		LiteralParser(const StringView& _str, const Profile& _profile = defaults() );
+
+		/// Returns true if literal was matched.
+		///
+		bool isOk() const;
+
+		/// Returns literal that was matched, or empty string view if there was no match.
+		///
+		const StringView& getLiteral() const;
+
+		/// Returns type of literal that was matched, or `Type::Unknown` if there was no match.
+		///
+		Type getType() const;
+
+	private:
+		StringView m_literal;
+		Type       m_type;
+		bool       m_ok;
+	};
+
+} // namespace bx
+
+#include "inline/literal-parser.inl"
+
+#endif // BX_LITERAL_PARSER_H_HEADER_GUARD

include/bx/scanner.h

@@ -29,9 +29,11 @@ namespace bx
 		///
 		enum class Class : uint8_t
 		{
-			Space,      //!< Run of whitespace characters.
+			Space,      //!< Run of whitespace characters, line terminators included.
+			SpaceHoriz, //!< Run of horizontal whitespace as `isSpaceHoriz` reads it, so up to but not including a line feed.
 			NonSpace,   //!< Run of non-whitespace characters.
-			Identifier, //!< Run of alphanumeric characters and `_`.
+			Identifier, //!< Identifier as `strIdentifier` reads one: alphanumeric characters and `_`, not starting with a digit.
+			Word,       //!< Word as `strWord` reads one: run of alphanumeric characters and `_`, a leading digit included.
 			EndOfLine,  //!< Everything up to, but not including, line terminator.
 			NewLine,    //!< Everything up to, and including, line terminator.
 		};
@@ -107,6 +109,19 @@ namespace bx
 		///
 		StringView accept(const StringView& _str);
 
+		/// Accept `_str` if input at cursor starts with it and it does not run into a word.
+		///
+		/// Unlike `accept`, which is a plain prefix test, a token ending in a word
+		/// character must be followed by something that is not one, so a keyword does not
+		/// match the head of a longer word: `acceptToken("if")` leaves `ifx` alone. For a
+		/// token ending in punctuation the two behave the same.
+		///
+		/// @param[in] _str Token to match.
+		///
+		/// @returns Accepted string view, or empty string view if it doesn't match.
+		///
+		StringView acceptToken(const StringView& _str);
+
 		/// Accept leading run of characters belonging to `_class`.
 		///
 		/// @param[in] _class Character class to match.
@@ -234,6 +249,15 @@ namespace bx
 		///
 		StringView getCursor(Cursor _which = Cursor::Current) const;
 
+		/// Returns input from cursor to end of input.
+		///
+		/// Where `getCursor` gives a position, this gives the range still to be scanned,
+		/// which is what another parser over the same text needs handed to it.
+		///
+		/// @returns String view from cursor to end of input.
+		///
+		StringView getTail() const;
+
 		/// Returns text between `_from` and current cursor position.
 		///
 		/// @param[in] _from Position to measure from, usually obtained by `getCursor`.

include/bx/sort.h

@@ -6,6 +6,7 @@
 #ifndef BX_SORT_H_HEADER_GUARD
 #define BX_SORT_H_HEADER_GUARD
 
+#include "allocator.h"
 #include "bx.h"
 #include "math.h"
 #include "string.h"
@@ -74,6 +75,52 @@ namespace bx
 	template<typename Ty>
 	void quickSort(Ty* _data, uint32_t _num, const ComparisonFn _fn = compareAscending<Ty>);
 
+	/// Performs sort (Insertion Sort algorithm).
+	///
+	/// It's stable, it sorts in place, and it doesn't allocate. Elements move by assignment rather
+	/// than by byte-wise swap, so unlike `quickSort` element type doesn't have to be trivially
+	/// move assignable. It's quadratic, and it's meant for short arrays, or as a building block of
+	/// `stableSort`.
+	///
+	/// @param _data Pointer to array data.
+	/// @param _num Number of elements.
+	/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
+	///   strict weak ordering.
+	///
+	template<typename Ty, typename LessFn>
+	void insertionSort(Ty* _data, uint32_t _num, LessFn _less);
+
+	/// Performs stable sort: elements that compare equal keep their relative order.
+	///
+	/// Insertion sort for short runs, and bottom-up merge for the rest, ping-ponging between input
+	/// and a single scratch buffer. Elements move by assignment rather than by byte-wise swap, so
+	/// unlike `quickSort` element type doesn't have to be trivially move assignable. An element
+	/// holding a small-string buffer, or anything else self-referential, is safe here and is not
+	/// safe there.
+	///
+	/// @param _allocator Allocator for scratch buffer. Nothing is allocated when there are few
+	///   enough elements for insertion sort alone.
+	/// @param _data Pointer to array data.
+	/// @param _num Number of elements.
+	/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
+	///   strict weak ordering.
+	///
+	/// @attention Element type must be default constructible and copy assignable.
+	///
+	template<typename Ty, typename LessFn>
+	void stableSort(AllocatorI* _allocator, Ty* _data, uint32_t _num, LessFn _less);
+
+	/// Performs stable sort over contiguous container.
+	///
+	/// @param _allocator Allocator for scratch buffer.
+	/// @param _container Container with `empty`, `size` and `operator[]`, such as
+	///   `tinystl::vector`.
+	/// @param _less Comparison function returning true if `_lhs` sorts before `_rhs`. It must be a
+	///   strict weak ordering.
+	///
+	template<typename ContainerT, typename LessFn>
+	void stableSort(AllocatorI* _allocator, ContainerT& _container, LessFn _less);
+
 	/// Performs reordering of duplicate elements in the array in the way that unique elements
 	/// are sorted to the front of array, and duplicates are after the return value index.
 	///

include/bx/string.h

@@ -420,6 +420,9 @@ namespace bx
 	/// Returns string view with whitespace characters trimmed from left.
 	StringView strLTrimSpace(const StringView& _str);
 
+	/// Returns string view with horizontal whitespace characters trimmed from left.
+	StringView strLTrimSpaceHoriz(const StringView& _str);
+
 	/// Returns string view with non-whitespace characters trimmed from left.
 	StringView strLTrimNonSpace(const StringView& _str);