- fbx: further work on broadphase tokenization.

code/FBXTokenizer.cpp

@@ -45,9 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER
 
-#include "FBXTokenizer.h"
 #include "ParsingUtils.h"
 
+#include "FBXTokenizer.h"
+#include "FBXUtil.h"
+
 namespace Assimp {
 namespace FBX {
 
@@ -70,6 +72,32 @@ Token::~Token()
 }
 
 
+namespace {
+
+// process a potential data token up to 'cur', adding it to 'output_tokens'. 
+// ------------------------------------------------------------------------------------------------
+void ProcessDataToken( TokenList& output_tokens, const char*& start, const char*& end,
+					  unsigned int line, 
+					  unsigned int column, 
+					  TokenType type = TokenType_DATA)
+{
+	if (start != end) {
+		// tokens should have no whitespace in them and [start,end] should
+		// properly delimit the valid range.
+		for (const char* c = start; c != end; ++c) {
+			if (IsSpaceOrNewLine(*c)) {
+				throw DeadlyImportError(Util::AddLineAndColumn("FBX-Tokenize","unexpected whitespace in token",line,column));
+			}
+		}
+
+		output_tokens.push_back(boost::make_shared<Token>(start,end,type,line,column));
+	}
+
+	start = end = NULL;
+}
+
+}
+
 // ------------------------------------------------------------------------------------------------
 void Tokenize(TokenList& output_tokens, const char* input)
 {
@@ -80,7 +108,9 @@ void Tokenize(TokenList& output_tokens, const char* input)
 	unsigned int column = 1;
 
 	bool comment = false;
+	bool in_double_quotes = false;
 	
+	const char* token_begin = NULL, *token_end = NULL;
 	for (const char* cur = input;*cur;++cur,++column) {
 		const char c = *cur;
 
@@ -97,30 +127,54 @@ void Tokenize(TokenList& output_tokens, const char* input)
 			continue;
 		}
 
+		if(in_double_quotes) {
+			if (c == '\"') {
+				in_double_quotes = false;
+				token_end = cur;
+				if (!token_begin) {
+					token_begin = cur;
+				}
+			}
+			continue;
+		}
+
 		switch(c)
 		{
+		case '\"':
+			in_double_quotes = true;
+			continue;
+
 		case ';':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			comment = true;
 			continue;
 
 		case '{':
+			ProcessDataToken(output_tokens,token_begin,token_end, line, column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_OPEN_BRACKET,line,column));
 			break;
 
 		case '}':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_CLOSE_BRACKET,line,column));
 			break;
 		
 		case ',':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
 			output_tokens.push_back(boost::make_shared<Token>(cur,cur+1,TokenType_COMMA,line,column));
 			break;
-		}
 
-		
-		if (IsSpaceOrNewLine(c)) {
-			//
+		case ':':
+			ProcessDataToken(output_tokens,token_begin,token_end,line,column, TokenType_KEY);
+			break;
+		}
+		
+		if (!IsSpaceOrNewLine(c)) {
+			token_end = cur;
+			if (!token_begin) {
+				token_begin = cur;
+			}
 		}
-		// XXX parse key and data elements
 	}
 }