From 056ea3486cb6aca02f2344111661e4e8f88ebb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Sun, 26 Jul 2026 08:32:02 -0700 Subject: [PATCH] Scanner: Cleanup. (#411) --- include/bx/inline/scanner.inl | 22 +++++++++++----------- include/bx/scanner.h | 26 ++++++++++++++------------ src/scanner.cpp | 8 +++++++- src/url.cpp | 32 ++++++++------------------------ tests/scanner_test.cpp | 20 ++++++++++---------- 5 files changed, 50 insertions(+), 58 deletions(-) diff --git a/include/bx/inline/scanner.inl b/include/bx/inline/scanner.inl index 879e363..bbb920b 100644 --- a/include/bx/inline/scanner.inl +++ b/include/bx/inline/scanner.inl @@ -12,14 +12,14 @@ namespace bx inline Scanner::Scanner(const StringView& _input) : m_input(_input) , m_tail(_input) - , m_line(0) + , m_line(1) { } inline void Scanner::reset() { m_tail = m_input; - m_line = 0; + m_line = 1; } inline StringView Scanner::acceptAll() @@ -43,7 +43,7 @@ namespace bx return moveBy(1); } - return StringView(); + return getCursor(); } inline StringView Scanner::accept(char _ch0, char _ch1) @@ -59,7 +59,7 @@ namespace bx } } - return StringView(); + return getCursor(); } template @@ -82,7 +82,7 @@ namespace bx return moveTo({ m_tail.getPtr(), _str.getLength() }); } - return StringView(); + return getCursor(); } inline StringView Scanner::accept(Class _class) @@ -98,7 +98,7 @@ namespace bx return moveBy(1); } - return StringView(); + return getCursor(); } inline StringView Scanner::acceptWhile(const StringView& _any) @@ -110,7 +110,7 @@ namespace bx return moveTo({ m_tail.getPtr(), input.getPtr() }); } - return StringView(); + return getCursor(); } inline StringView Scanner::acceptWhile(CharTestFn _fn) @@ -136,7 +136,7 @@ namespace bx return moveTo({ m_tail.getPtr(), result.getPtr() }); } - return result; + return getCursor(); } inline StringView Scanner::acceptUntil(Class _class) @@ -148,7 +148,7 @@ namespace bx inline StringView Scanner::peek() const { return m_tail.isEmpty() - ? StringView() + ? getCursor() : StringView(m_tail.getPtr(), m_tail.getPtr() + 1) ; } @@ -161,7 +161,7 @@ namespace bx return StringView(m_tail.getPtr(), m_tail.getPtr() + 1); } - return StringView(); + return getCursor(); } inline StringView Scanner::peek(const StringView& _str) const @@ -171,7 +171,7 @@ namespace bx return StringView(m_tail.getPtr(), _str.getLength() ); } - return StringView(); + return getCursor(); } inline StringView Scanner::peek(Class _class) const diff --git a/include/bx/scanner.h b/include/bx/scanner.h index bbcbf63..01e2fcf 100644 --- a/include/bx/scanner.h +++ b/include/bx/scanner.h @@ -16,7 +16,9 @@ namespace bx /// moves. `accept*` functions advance cursor when they match, `peek*` functions perform the /// same test without moving cursor. /// - /// All returned string views point into input string, and no copies are made. + /// All returned string views point into input string, and no copies are made. This includes + /// empty string views returned when nothing matched, which point at cursor rather than being + /// default constructed. They can be used with `seek` and `between`. /// /// @attention Scanner doesn't own input string. Input string must outlive Scanner. /// @@ -142,11 +144,11 @@ namespace bx /// /// @param[in] _find String to search for. /// - /// @returns Accepted string view, or empty string view if `_find` is not found, or it's - /// already at cursor. + /// @returns Accepted string view, or empty string view at cursor if `_find` is not found, + /// or it's already at cursor. /// - /// @attention Returned string view is empty in both cases, and cursor doesn't move. - /// Use `getCursor` and `between` to capture text that can legitimately be empty. + /// @attention Returned string view is empty in both cases, and cursor doesn't move. If + /// these two cases have to be told apart, test for `_find` separately with `peek`. /// StringView acceptUntil(const StringView& _find); @@ -208,7 +210,7 @@ namespace bx /// bool seek(int32_t _bytes); - /// Returns zero-based line number cursor is on. + /// Returns one-based line number cursor is on. /// /// @returns Line number. /// @@ -226,9 +228,9 @@ namespace bx /// /// @returns Zero length string view at requested position. /// - /// @remarks Unlike default constructed `StringView`, returned string view keeps pointer - /// into input string even though it's empty. Use it with `between` to capture text that - /// can legitimately be empty. + /// @remarks Returned string view keeps pointer into input string even though it's empty, + /// and it can be passed to `seek` to return to this position later, or to `between` to + /// capture text that can legitimately be empty. /// StringView getCursor(Cursor _which = Cursor::Current) const; @@ -250,10 +252,10 @@ namespace bx private: /// Move cursor to `_to`, and update line number. /// - /// @param[in] _to Position to move cursor to. + /// @param[in] _to Position to move cursor to. Empty string view is treated as cursor + /// position obtained by `getCursor` rather than as span, and cursor moves to it. /// - /// @returns String view between old and new cursor position, or empty string view if `_to` - /// is empty. + /// @returns String view between old and new cursor position. /// StringView moveTo(const StringView& _to); diff --git a/src/scanner.cpp b/src/scanner.cpp index 41e3206..8cb923b 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -71,7 +71,13 @@ namespace bx return result; } - return StringView(); + if (_to.getPtr() != m_tail.getPtr() + && contain(m_input, _to) ) + { + moveBy(int32_t(_to.getPtr() - m_tail.getPtr() ) ); + } + + return getCursor(); } StringView Scanner::strFunc(Class _class) const diff --git a/src/url.cpp b/src/url.cpp index a132aca..ca6c3e6 100644 --- a/src/url.cpp +++ b/src/url.cpp @@ -51,9 +51,7 @@ namespace bx Scanner scanner(_url); - const StringView schemeBegin = scanner.getCursor(); - scanner.acceptUntil("://"); - const StringView scheme = scanner.between(schemeBegin); + const StringView scheme = scanner.acceptUntil("://"); const bool hasScheme = !scanner.accept("://").isEmpty(); @@ -67,9 +65,7 @@ namespace bx m_tokens[Scheme].set(scheme); } - const StringView authorityBegin = scanner.getCursor(); - scanner.acceptWhile(isNotSlash); - const StringView authority = scanner.between(authorityBegin); + const StringView authority = scanner.acceptWhile(isNotSlash); const bool hasPath = !scanner.peek('/').isEmpty(); @@ -81,22 +77,16 @@ namespace bx if (hasPath) { - const StringView pathBegin = scanner.getCursor(); - scanner.acceptWhile(isNotQueryOrFragment); - m_tokens[Path].set(scanner.between(pathBegin) ); + m_tokens[Path].set(scanner.acceptWhile(isNotQueryOrFragment) ); if (!scanner.accept('?').isEmpty() ) { - const StringView queryBegin = scanner.getCursor(); - scanner.acceptWhile(isNotFragment); - m_tokens[Query].set(scanner.between(queryBegin) ); + m_tokens[Query].set(scanner.acceptWhile(isNotFragment) ); } if (!scanner.accept('#').isEmpty() ) { - const StringView fragmentBegin = scanner.getCursor(); - scanner.acceptWhile(isNotQuery); - m_tokens[Fragment].set(scanner.between(fragmentBegin) ); + m_tokens[Fragment].set(scanner.acceptWhile(isNotQuery) ); } // Anything left over is a query following a fragment. @@ -108,17 +98,13 @@ namespace bx Scanner authorityScanner(authority); - const StringView userInfoBegin = authorityScanner.getCursor(); - authorityScanner.acceptUntil("@"); - const StringView userInfo = authorityScanner.between(userInfoBegin); + const StringView userInfo = authorityScanner.acceptUntil("@"); if (!authorityScanner.accept('@').isEmpty() ) { Scanner userInfoScanner(userInfo); - const StringView userNameBegin = userInfoScanner.getCursor(); - userInfoScanner.acceptWhile(isNotColon); - m_tokens[UserName].set(userInfoScanner.between(userNameBegin) ); + m_tokens[UserName].set(userInfoScanner.acceptWhile(isNotColon) ); if (!userInfoScanner.accept(':').isEmpty() ) { @@ -126,9 +112,7 @@ namespace bx } } - const StringView hostBegin = authorityScanner.getCursor(); - authorityScanner.acceptWhile(isNotColon); - m_tokens[Host].set(authorityScanner.between(hostBegin) ); + m_tokens[Host].set(authorityScanner.acceptWhile(isNotColon) ); if (!authorityScanner.accept(':').isEmpty() ) { diff --git a/tests/scanner_test.cpp b/tests/scanner_test.cpp index 2130099..2ba5b69 100644 --- a/tests/scanner_test.cpp +++ b/tests/scanner_test.cpp @@ -14,7 +14,7 @@ namespace bx Scanner scanner(_str); scanner.acceptAll(); - return scanner.getLine(); + return scanner.getLine() - 1; } void printLines(const StringView& _str) @@ -79,27 +79,27 @@ TEST_CASE("Scanner", "[scanner]") REQUIRE(!scanner.isDone() ); REQUIRE("xyz" == scanner.accept("xyz") ); - REQUIRE(0 == scanner.getLine() ); - REQUIRE("\n " == scanner.acceptUntil(bx::Scanner::Class::Space) ); REQUIRE(1 == scanner.getLine() ); + REQUIRE("\n " == scanner.acceptUntil(bx::Scanner::Class::Space) ); + REQUIRE(2 == scanner.getLine() ); scanner.acceptAll(); REQUIRE(scanner.accept().isEmpty() ); - REQUIRE(numLines == scanner.getLine() ); + REQUIRE(numLines + 1 == scanner.getLine() ); REQUIRE(scanner.isDone() ); REQUIRE(scanner.seek(start) ); - REQUIRE(0 == scanner.getLine() ); + REQUIRE(1 == scanner.getLine() ); REQUIRE(!scanner.isDone() ); REQUIRE(scanner.seek(INT32_MAX) ); - REQUIRE(numLines == scanner.getLine() ); + REQUIRE(numLines + 1 == scanner.getLine() ); REQUIRE(scanner.isDone() ); REQUIRE(scanner.seek(INT32_MIN) ); - REQUIRE(0 == scanner.getLine() ); + REQUIRE(1 == scanner.getLine() ); REQUIRE(!scanner.isDone() ); } @@ -206,7 +206,7 @@ TEST_CASE("Scanner.getColumn", "[scanner]") REQUIRE(4 == sc.getColumn() ); sc.accept('\n'); - REQUIRE(1 == sc.getLine() ); + REQUIRE(2 == sc.getLine() ); REQUIRE(1 == sc.getColumn() ); sc.accept(); REQUIRE(2 == sc.getColumn() ); @@ -237,11 +237,11 @@ TEST_CASE("Scanner.reset", "[scanner]") sc.acceptAll(); REQUIRE(sc.isDone() ); - REQUIRE(2 == sc.getLine() ); + REQUIRE(3 == sc.getLine() ); sc.reset(); REQUIRE(!sc.isDone() ); - REQUIRE(0 == sc.getLine() ); + REQUIRE(1 == sc.getLine() ); REQUIRE(1 == sc.getColumn() ); REQUIRE("abc" == sc.accept(bx::Scanner::Class::Identifier) );