diff --git a/code/FBXTokenizer.cpp b/code/FBXTokenizer.cpp index f210c2da6..0bd8e4dd6 100644 --- a/code/FBXTokenizer.cpp +++ b/code/FBXTokenizer.cpp @@ -45,9 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER -#include "FBXTokenizer.h" #include "ParsingUtils.h" +#include "FBXTokenizer.h" +#include "FBXUtil.h" + namespace Assimp { namespace FBX { @@ -70,6 +72,32 @@ Token::~Token() } +namespace { + +// process a potential data token up to 'cur', adding it to 'output_tokens'. +// ------------------------------------------------------------------------------------------------ +void ProcessDataToken( TokenList& output_tokens, const char*& start, const char*& end, + unsigned int line, + unsigned int column, + TokenType type = TokenType_DATA) +{ + if (start != end) { + // tokens should have no whitespace in them and [start,end] should + // properly delimit the valid range. + for (const char* c = start; c != end; ++c) { + if (IsSpaceOrNewLine(*c)) { + throw DeadlyImportError(Util::AddLineAndColumn("FBX-Tokenize","unexpected whitespace in token",line,column)); + } + } + + output_tokens.push_back(boost::make_shared(start,end,type,line,column)); + } + + start = end = NULL; +} + +} + // ------------------------------------------------------------------------------------------------ void Tokenize(TokenList& output_tokens, const char* input) { @@ -80,7 +108,9 @@ void Tokenize(TokenList& output_tokens, const char* input) unsigned int column = 1; bool comment = false; + bool in_double_quotes = false; + const char* token_begin = NULL, *token_end = NULL; for (const char* cur = input;*cur;++cur,++column) { const char c = *cur; @@ -97,30 +127,54 @@ void Tokenize(TokenList& output_tokens, const char* input) continue; } + if(in_double_quotes) { + if (c == '\"') { + in_double_quotes = false; + token_end = cur; + if (!token_begin) { + token_begin = cur; + } + } + continue; + } + switch(c) { + case '\"': + in_double_quotes = true; + continue; + case ';': + ProcessDataToken(output_tokens,token_begin,token_end,line,column); comment = true; continue; case '{': + ProcessDataToken(output_tokens,token_begin,token_end, line, column); output_tokens.push_back(boost::make_shared(cur,cur+1,TokenType_OPEN_BRACKET,line,column)); break; case '}': + ProcessDataToken(output_tokens,token_begin,token_end,line,column); output_tokens.push_back(boost::make_shared(cur,cur+1,TokenType_CLOSE_BRACKET,line,column)); break; case ',': + ProcessDataToken(output_tokens,token_begin,token_end,line,column); output_tokens.push_back(boost::make_shared(cur,cur+1,TokenType_COMMA,line,column)); break; - } - - if (IsSpaceOrNewLine(c)) { - // + case ':': + ProcessDataToken(output_tokens,token_begin,token_end,line,column, TokenType_KEY); + break; + } + + if (!IsSpaceOrNewLine(c)) { + token_end = cur; + if (!token_begin) { + token_begin = cur; + } } - // XXX parse key and data elements } } diff --git a/code/FBXTokenizer.h b/code/FBXTokenizer.h index e37210c7c..1a7899fca 100644 --- a/code/FBXTokenizer.h +++ b/code/FBXTokenizer.h @@ -118,7 +118,8 @@ typedef std::vector< boost::shared_ptr > TokenList; * Skips over comments and generates line and column numbers. * * @param output_tokens Receives a list of all tokens in the input data. - * @param input_buffer Textual input buffer to be processed, 0-terminated. */ + * @param input_buffer Textual input buffer to be processed, 0-terminated. + * @throw DeadlyImportError if something goes wrong */ void Tokenize(TokenList& output_tokens, const char* input); diff --git a/code/FBXUtil.cpp b/code/FBXUtil.cpp new file mode 100644 index 000000000..66647cb66 --- /dev/null +++ b/code/FBXUtil.cpp @@ -0,0 +1,66 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2012, assimp team +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +/** @file FBXUtil.cpp + * @brief Implementation of internal FBX utility functions + */ +#include "AssimpPCH.h" + +#include "FBXUtil.h" +#include "TinyFormatter.h" + +#ifndef ASSIMP_BUILD_NO_FBX_IMPORTER + +namespace Assimp { +namespace FBX { +namespace Util { + +// ------------------------------------------------------------------------------------------------ +std::string AddLineAndColumn(const std::string& prefix, const std::string& text, unsigned int line, unsigned int column) +{ + return static_cast( (Formatter::format(),prefix,"(line ",line,", col ",column,") ",text) ); +} + +} // !Util +} // !FBX +} // !Assimp + +#endif + diff --git a/code/FBXUtil.h b/code/FBXUtil.h new file mode 100644 index 000000000..87bb86f70 --- /dev/null +++ b/code/FBXUtil.h @@ -0,0 +1,67 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2012, assimp team +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ + +/** @file FBXUtil.h + * @brief FBX utility functions for internal use + */ +#ifndef INCLUDED_AI_FBX_UTIL_H +#define INCLUDED_AI_FBX_UTIL_H + +#include +#include "FBXCompileConfig.h" + +namespace Assimp { +namespace FBX { +namespace Util { + +/** Format log/error messages using a given line location in the source file. + * + * @param prefix Message prefix to be preprended to the location info. + * @param text Message text + * @param line Line index, 1-based + * @param column Colum index, 1-based + * @return A string of the following format: {prefix} (line {line}, col {column}) {text}*/ +std::string AddLineAndColumn(const std::string& prefix, const std::string& text, unsigned int line, unsigned int column); + +} +} +} + +#endif // ! INCLUDED_AI_FBX_UTIL_H diff --git a/workspaces/vc9/assimp.vcproj b/workspaces/vc9/assimp.vcproj index bd781cef5..431f4efc6 100644 --- a/workspaces/vc9/assimp.vcproj +++ b/workspaces/vc9/assimp.vcproj @@ -2083,6 +2083,14 @@ RelativePath="..\..\code\FBXTokenizer.h" > + + + +