Added scanner. (#410)

This commit is contained in:
Branimir Karadžić
2026-07-25 12:23:58 -07:00
committed by GitHub
parent cd6720ce9a
commit 9916e720fc
21 changed files with 2157 additions and 1899 deletions
+228
View File
@@ -0,0 +1,228 @@
/*
* Copyright 2010-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#ifndef BX_SCANNER_H_HEADER_GUARD
# error "Must be included from bx/scanner.h!"
#endif // BX_SCANNER_H_HEADER_GUARD
namespace bx
{
inline Scanner::Scanner(const StringView& _input)
: m_input(_input)
, m_tail(_input)
, m_line(0)
{
}
inline void Scanner::reset()
{
m_tail = m_input;
m_line = 0;
}
inline StringView Scanner::acceptAll()
{
StringView result = m_tail;
moveBy(result.getLength() );
return result;
}
inline StringView Scanner::accept()
{
return moveBy(1);
}
inline StringView Scanner::accept(char _ch)
{
if (!m_tail.isEmpty()
&& _ch == *m_tail.getPtr() )
{
return moveBy(1);
}
return StringView();
}
inline StringView Scanner::accept(char _ch0, char _ch1)
{
if (!m_tail.isEmpty() )
{
const char ch = *m_tail.getPtr();
if (ch == _ch0
|| ch == _ch1)
{
return moveBy(1);
}
}
return StringView();
}
template<typename ...Args>
inline StringView Scanner::accept(char _ch0, char _ch1, Args... _args)
{
StringView result = accept(_ch0, _ch1);
if (!result.isEmpty() )
{
return result;
}
return accept(_args...);
}
inline StringView Scanner::accept(const StringView& _str)
{
if (hasPrefix(m_tail, _str) )
{
return moveTo({ m_tail.getPtr(), _str.getLength() });
}
return StringView();
}
inline StringView Scanner::accept(Class _class)
{
return moveTo({ m_tail.getPtr(), strFunc(_class).getPtr() });
}
inline StringView Scanner::accept(CharTestFn _fn)
{
if (!m_tail.isEmpty()
&& _fn(*m_tail.getPtr() ) )
{
return moveBy(1);
}
return StringView();
}
inline StringView Scanner::acceptWhile(const StringView& _any)
{
const StringView input = strLTrim(m_tail, _any);
if (input.getPtr() != m_tail.getPtr() )
{
return moveTo({ m_tail.getPtr(), input.getPtr() });
}
return StringView();
}
inline StringView Scanner::acceptWhile(CharTestFn _fn)
{
const char* ptr = m_tail.getPtr();
const char* term = m_tail.getTerm();
while (ptr != term
&& _fn(*ptr) )
{
++ptr;
}
return moveTo({ m_tail.getPtr(), ptr });
}
inline StringView Scanner::acceptUntil(const StringView& _find)
{
StringView result = strFind(m_tail, _find);
if (!result.isEmpty() )
{
return moveTo({ m_tail.getPtr(), result.getPtr() });
}
return result;
}
inline StringView Scanner::acceptUntil(Class _class)
{
StringView result = strFunc(_class);
return moveTo({ m_tail.getPtr(), result.getPtr() });
}
inline StringView Scanner::peek() const
{
return m_tail.isEmpty()
? StringView()
: StringView(m_tail.getPtr(), m_tail.getPtr() + 1)
;
}
inline StringView Scanner::peek(char _ch) const
{
if (!m_tail.isEmpty()
&& _ch == *m_tail.getPtr() )
{
return StringView(m_tail.getPtr(), m_tail.getPtr() + 1);
}
return StringView();
}
inline StringView Scanner::peek(const StringView& _str) const
{
if (hasPrefix(m_tail, _str) )
{
return StringView(m_tail.getPtr(), _str.getLength() );
}
return StringView();
}
inline StringView Scanner::peek(Class _class) const
{
return StringView(m_tail.getPtr(), strFunc(_class).getPtr() );
}
inline bool Scanner::seek(int32_t _bytes)
{
moveBy(_bytes);
return true;
}
inline uint32_t Scanner::getLine() const
{
return m_line;
}
inline StringView Scanner::between(const StringView& _from) const
{
const char* a = _from.getPtr();
const char* b = m_tail.getPtr();
return a <= b ? StringView(a, b) : StringView(b, a);
}
inline bool Scanner::isDone() const
{
return m_tail.isEmpty();
}
inline LineReader::LineReader(const StringView& _str)
: m_scanner(_str)
, m_line(0)
{
}
inline void LineReader::reset()
{
m_scanner.reset();
m_line = 0;
}
inline uint32_t LineReader::getLine() const
{
return m_line;
}
inline bool LineReader::isDone() const
{
return m_scanner.isDone();
}
} // namespace bx
+110 -39
View File
@@ -461,50 +461,121 @@ namespace bx
return StringView(m_ptr, m_len);
}
inline BX_CONSTEXPR_FUNC bool isInRange(char _ch, char _from, char _to)
{
return unsigned(_ch - _from) <= unsigned(_to-_from);
}
inline BX_CONSTEXPR_FUNC bool isSpace(char _ch)
{
return ' ' == _ch // Space.
|| '\t' == _ch // Horizontal tab.
|| '\n' == _ch // Line feed / new line.
|| '\r' == _ch // Carriage return.
|| '\v' == _ch // Vertical tab.
|| '\f' == _ch // Form feed / new page.
;
}
inline BX_CONSTEXPR_FUNC bool isSpaceHoriz(char _ch)
{
return false
|| ' ' == _ch
|| '\t' == _ch
|| '\r' == _ch
;
}
inline BX_CONSTEXPR_FUNC bool isUpper(char _ch)
{
return isInRange(_ch, 'A', 'Z');
}
inline BX_CONSTEXPR_FUNC char toUpper(char _ch)
{
return _ch - (isLower(_ch) ? 0x20 : 0);
}
inline BX_CONSTEXPR_FUNC bool isLower(char _ch)
{
return isInRange(_ch, 'a', 'z');
}
inline BX_CONSTEXPR_FUNC char toLower(char _ch)
{
return _ch + (isUpper(_ch) ? 0x20 : 0);
}
inline BX_CONSTEXPR_FUNC bool isAlpha(char _ch)
{
return isLower(_ch) || isUpper(_ch);
}
inline BX_CONSTEXPR_FUNC bool isNumeric(char _ch)
{
return isInRange(_ch, '0', '9');
}
inline BX_CONSTEXPR_FUNC bool isAlphaNum(char _ch)
{
return false
|| isAlpha(_ch)
|| isNumeric(_ch)
;
}
inline BX_CONSTEXPR_FUNC bool isHexNum(char _ch)
{
return false
|| isInRange(toLower(_ch), 'a', 'f')
|| isNumeric(_ch)
;
}
inline BX_CONSTEXPR_FUNC bool isOctNum(char _ch)
{
return isInRange(_ch, '0', '7');
}
inline BX_CONSTEXPR_FUNC bool isBinNum(char _ch)
{
return isInRange(_ch, '0', '1');
}
inline BX_CONSTEXPR_FUNC bool isPrint(char _ch)
{
return isInRange(_ch, ' ', '~');
}
inline BX_CONSTEXPR_FUNC bool isIdentStart(char _ch)
{
return false
|| isAlpha(_ch)
|| '_' == _ch
;
}
inline BX_CONSTEXPR_FUNC bool isIdentChar(char _ch)
{
return false
|| isAlphaNum(_ch)
|| '_' == _ch
;
}
inline BX_CONSTEXPR_FUNC bool isPathSeparator(char _ch)
{
return false
|| '/' == _ch
|| '\\' == _ch
;
}
inline StringView strSubstr(const StringView& _str, int32_t _start, int32_t _len)
{
return StringView(_str, _start, _len);
}
inline LineReader::LineReader(const StringView& _str)
: m_str(_str)
{
reset();
}
inline void LineReader::reset()
{
m_curr = m_str;
m_line = 0;
}
inline StringView LineReader::next()
{
if (m_curr.getPtr() != m_str.getTerm() )
{
++m_line;
StringView curr(m_curr);
m_curr = strFindNl(m_curr);
StringView line(curr.getPtr(), m_curr.getPtr() );
return strRTrim(strRTrim(line, "\n"), "\r");
}
return m_curr;
}
inline bool LineReader::isDone() const
{
return m_curr.getPtr() == m_str.getTerm();
}
inline uint32_t LineReader::getLine() const
{
return m_line;
}
inline constexpr int32_t strLen(const StringView& _str, int32_t _max)
{
return min(_str.getLength(), _max);
+341
View File
@@ -0,0 +1,341 @@
/*
* Copyright 2011-2026 Branimir Karadzic. All rights reserved.
* License: https://github.com/bkaradzic/bx/blob/master/LICENSE
*/
#ifndef BX_SCANNER_H_HEADER_GUARD
#define BX_SCANNER_H_HEADER_GUARD
#include <bx/string.h>
namespace bx
{
/// Forward-only cursor over a string view, used to tokenize text input.
///
/// Scanner keeps a cursor inside immutable input string, and tracks line number as cursor
/// moves. `accept*` functions advance cursor when they match, `peek*` functions perform the
/// same test without moving cursor.
///
/// All returned string views point into input string, and no copies are made.
///
/// @attention Scanner doesn't own input string. Input string must outlive Scanner.
///
class Scanner
{
public:
/// Character class matched by `accept`, `peek`, and `acceptUntil`.
///
enum class Class : uint8_t
{
Space, //!< Run of whitespace characters.
NonSpace, //!< Run of non-whitespace characters.
Identifier, //!< Run of alphanumeric characters and `_`.
EndOfLine, //!< Everything up to, but not including, line terminator.
NewLine, //!< Everything up to, and including, line terminator.
};
/// Position inside input string returned by `getCursor`.
///
enum class Cursor : uint8_t
{
Current, //!< Current cursor position.
LineStart, //!< First character of line cursor is on.
LineEnd, //!< Position of `\n` that terminates line cursor is on, or end of input.
};
/// Default constructor is not available. Scanner must be constructed with input string.
///
Scanner() = delete;
/// Constructor.
///
/// @param[in] _input Input string to scan. It's not copied, and it must outlive Scanner.
///
Scanner(const StringView& _input);
/// Rewind cursor to the beginning of input string, and reset line counter.
///
void reset();
/// Accept everything from cursor to the end of input string.
///
/// @returns Accepted string view, or empty string view if input is exhausted.
///
StringView acceptAll();
/// Accept single character.
///
/// @returns Accepted character, or empty string view if input is exhausted.
///
StringView accept();
/// Accept single character if it matches `_ch`.
///
/// @param[in] _ch Character to match.
///
/// @returns Accepted character, or empty string view if it doesn't match.
///
StringView accept(char _ch);
/// Accept single character if it matches any of the characters passed in.
///
/// @param[in] _ch0 First character to match.
/// @param[in] _ch1 Second character to match.
///
/// @returns Accepted character, or empty string view if it doesn't match.
///
StringView accept(char _ch0, char _ch1);
/// Accept single character if it matches any of the characters passed in.
///
/// @param[in] _ch0 First character to match.
/// @param[in] _ch1 Second character to match.
/// @param[in] _args Additional characters to match.
///
/// @returns Accepted character, or empty string view if it doesn't match.
///
template<typename ...Args>
StringView accept(char _ch0, char _ch1, Args... _args);
/// Accept `_str` if input at cursor starts with it.
///
/// @param[in] _str String to match.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
StringView accept(const StringView& _str);
/// Accept leading run of characters belonging to `_class`.
///
/// @param[in] _class Character class to match.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
StringView accept(Class _class);
/// Accept single character if `_fn` returns true for it.
///
/// @param[in] _fn Character test function.
///
/// @returns Accepted character, or empty string view if it doesn't match.
///
StringView accept(CharTestFn _fn);
/// Accept characters as long as they are contained in `_any`.
///
/// @param[in] _any Set of characters to match.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
StringView acceptWhile(const StringView& _any);
/// Accept characters as long as `_fn` returns true for them.
///
/// @param[in] _fn Character test function.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
StringView acceptWhile(CharTestFn _fn);
/// Accept characters until `_find` is found. Cursor stops at `_find`, and `_find` is not
/// accepted.
///
/// @param[in] _find String to search for.
///
/// @returns Accepted string view, or empty string view if `_find` is not found, or it's
/// already at cursor.
///
/// @attention Returned string view is empty in both cases, and cursor doesn't move.
/// Use `getCursor` and `between` to capture text that can legitimately be empty.
///
StringView acceptUntil(const StringView& _find);
/// Accept leading run of characters belonging to `_class`.
///
/// @param[in] _class Character class to match.
///
/// @returns Accepted string view, or empty string view if it doesn't match.
///
/// @remarks Same as `accept(Class)`. It's provided for readability when used with
/// `Class::EndOfLine` and `Class::NewLine`.
///
StringView acceptUntil(Class _class);
/// Peek at single character without moving cursor.
///
/// @returns Next character, or empty string view if input is exhausted.
///
StringView peek() const;
/// Peek at single character without moving cursor, if it matches `_ch`.
///
/// @param[in] _ch Character to match.
///
/// @returns Next character, or empty string view if it doesn't match.
///
StringView peek(char _ch) const;
/// Peek at `_str` without moving cursor, if input at cursor starts with it.
///
/// @param[in] _str String to match.
///
/// @returns Matched string view, or empty string view if it doesn't match.
///
StringView peek(const StringView& _str) const;
/// Peek at leading run of characters belonging to `_class`, without moving cursor.
///
/// @param[in] _class Character class to match.
///
/// @returns Matched string view. It's zero length if it doesn't match.
///
StringView peek(Class _class) const;
/// Move cursor to `_to`. Line number is adjusted accordingly, and cursor can move backward.
///
/// @param[in] _to Position to move cursor to. It must be a string view into input string.
///
/// @returns True if cursor moved, or false if `_to` is not a string view into input string.
///
bool seek(const StringView& _to);
/// Move cursor by `_bytes`. Negative value moves cursor backward. Line number is adjusted
/// accordingly, and value is clamped to input string bounds.
///
/// @param[in] _bytes Number of bytes to move cursor by.
///
/// @returns Always true.
///
bool seek(int32_t _bytes);
/// Returns zero-based line number cursor is on.
///
/// @returns Line number.
///
uint32_t getLine() const;
/// Returns one-based column number cursor is on.
///
/// @returns Column number.
///
uint32_t getColumn() const;
/// Returns cursor position as zero length string view.
///
/// @param[in] _which Position to return.
///
/// @returns Zero length string view at requested position.
///
/// @remarks Unlike default constructed `StringView`, returned string view keeps pointer
/// into input string even though it's empty. Use it with `between` to capture text that
/// can legitimately be empty.
///
StringView getCursor(Cursor _which = Cursor::Current) const;
/// Returns text between `_from` and current cursor position.
///
/// @param[in] _from Position to measure from, usually obtained by `getCursor`.
///
/// @returns String view between `_from` and cursor. Order of arguments doesn't matter,
/// `_from` can be either before or after cursor.
///
StringView between(const StringView& _from) const;
/// Returns true if input string is exhausted.
///
/// @returns True if cursor reached end of input string, otherwise returns false.
///
bool isDone() const;
private:
/// Move cursor to `_to`, and update line number.
///
/// @param[in] _to Position to move cursor to.
///
/// @returns String view between old and new cursor position, or empty string view if `_to`
/// is empty.
///
StringView moveTo(const StringView& _to);
/// Move cursor by `_bytes` clamped to input string bounds, and update line number.
///
/// @param[in] _bytes Number of bytes to move cursor by.
///
/// @returns String view between old and new cursor position.
///
StringView moveBy(int32_t _bytes);
/// Returns end of leading run of characters belonging to `_class`.
///
/// @param[in] _class Character class to match.
///
/// @returns String view starting past the run, and ending at end of input string.
///
StringView strFunc(Class _class) const;
/// Count number of line terminators in `_str`.
///
/// @param[in] _str String to count line terminators in.
///
/// @returns Number of lines.
///
static uint32_t countLines(const StringView& _str);
const StringView m_input;
StringView m_tail;
uint32_t m_line;
};
/// Splits string into lines.
///
/// Line terminator is `\n` or `\r\n`, and it's not part of returned line. Trailing `\r` left
/// by malformed input is trimmed.
///
/// All returned string views point into input string, and no copies are made.
///
/// @attention LineReader doesn't own input string. Input string must outlive LineReader.
///
class LineReader
{
public:
/// Constructor.
///
/// @param[in] _str Input string to read lines from. It's not copied, and it must outlive
/// LineReader.
///
LineReader(const StringView& _str);
/// Rewind to the beginning of input string, and reset line counter.
///
void reset();
/// Read next line.
///
/// @returns Next line without line terminator. Line can be empty, and returned string view
/// keeps pointer into input string. If input is exhausted, zero length string view at
/// end of input string is returned.
///
StringView next();
/// Returns one-based line number of line returned by the last `next` call.
///
/// @returns Line number, or 0 if `next` was not called yet.
///
uint32_t getLine() const;
/// Returns true if input string is exhausted.
///
/// @returns True if all lines are read, otherwise returns false.
///
bool isDone() const;
private:
Scanner m_scanner;
uint32_t m_line;
};
} // namespace bx
#include "inline/scanner.inl"
#endif // BX_SCANNER_H_HEADER_GUARD
+3 -1
View File
@@ -46,8 +46,10 @@ namespace bx
private:
Settings();
struct Ini;
AllocatorI* m_allocator;
void* m_ini;
Ini* m_ini;
};
///
+30 -35
View File
@@ -279,6 +279,8 @@ namespace bx
int32_t m_capacity;
};
typedef bool (*CharTestFn)(char _ch);
/// Returns true if character is part of white space set.
///
/// White space set is:
@@ -289,55 +291,73 @@ namespace bx
/// '\v' - Vertical tab.
/// '\f' - Form feed / new page.
///
bool isSpace(char _ch);
BX_CONSTEXPR_FUNC bool isSpace(char _ch);
/// Returns true if string view contains only space characters.
bool isSpace(const StringView& _str);
/// Returns true if character is horizontal white space.
BX_CONSTEXPR_FUNC bool isSpaceHoriz(char _ch);
/// Returns true if character is uppercase.
bool isUpper(char _ch);
BX_CONSTEXPR_FUNC bool isUpper(char _ch);
/// Returns true if string view contains only uppercase characters.
bool isUpper(const StringView& _str);
/// Returns true if character is lowercase.
bool isLower(char _ch);
BX_CONSTEXPR_FUNC bool isLower(char _ch);
/// Returns true if string view contains only lowercase characters.
bool isLower(const StringView& _str);
/// Returns true if character is part of alphabet set.
bool isAlpha(char _ch);
BX_CONSTEXPR_FUNC bool isAlpha(char _ch);
/// Returns true if string view contains only alphabet characters.
bool isAlpha(const StringView& _str);
/// Returns true if character is part of numeric set.
bool isNumeric(char _ch);
BX_CONSTEXPR_FUNC bool isNumeric(char _ch);
/// Returns true if string view contains only numeric characters.
bool isNumeric(const StringView& _str);
/// Returns true if character is part of alpha numeric set.
bool isAlphaNum(char _ch);
BX_CONSTEXPR_FUNC bool isAlphaNum(char _ch);
/// Returns true if string view contains only alphanumeric characters.
bool isAlphaNum(const StringView& _str);
/// Returns true if character is part of hexadecimal set.
bool isHexNum(char _ch);
BX_CONSTEXPR_FUNC bool isHexNum(char _ch);
/// Returns true if string view contains only hexadecimal characters.
bool isHexNum(const StringView& _str);
///
BX_CONSTEXPR_FUNC bool isOctNum(char _ch);
///
BX_CONSTEXPR_FUNC bool isBinNum(char _ch);
///
BX_CONSTEXPR_FUNC bool isIdentStart(char _ch);
///
BX_CONSTEXPR_FUNC bool isIdentChar(char _ch);
///
BX_CONSTEXPR_FUNC bool isPathSeparator(char _ch);
/// Returns true if character is printable.
bool isPrint(char _ch);
BX_CONSTEXPR_FUNC bool isPrint(char _ch);
/// Returns true if string vieww contains only printable characters.
bool isPrint(const StringView& _str);
/// Returns lower case character representing _ch.
char toLower(char _ch);
BX_CONSTEXPR_FUNC char toLower(char _ch);
/// Lower case string in place assuming length passed is valid.
void toLowerUnsafe(char* _inOutStr, int32_t _len);
@@ -346,7 +366,7 @@ namespace bx
void toLower(char* _inOutStr, int32_t _max = INT32_MAX);
/// Returns upper case character representing _ch.
char toUpper(char _ch);
BX_CONSTEXPR_FUNC char toUpper(char _ch);
/// Upper case string in place assuming length passed is valid.
void toUpperUnsafe(char* _inOutStr, int32_t _len);
@@ -557,31 +577,6 @@ namespace bx
/// Converts string to 64-bit unsigned long long value.
bool fromString(unsigned long long* _out, const StringView& _str);
///
class LineReader
{
public:
///
LineReader(const StringView& _str);
///
void reset();
///
StringView next();
///
bool isDone() const;
///
uint32_t getLine() const;
private:
const StringView m_str;
StringView m_curr;
uint32_t m_line;
};
} // namespace bx
#include "inline/string.inl"